Accurate, fast FP4 KV-cache decoding through algorithm-system co-design.
Initial public release
DD-DuDa62d5cc227137
diff --git a/.gitignore b/.gitignorenew file mode 100644index 0000000..57598da--- /dev/null+++ b/.gitignore@@ -0,0 +1,13 @@+__pycache__/+.pytest_cache/+*.py[cod]+*.egg-info/+_out/++# Local environments, models, sidecars, run outputs, and LCB checkout.+_local/++# Datasets are staged by tools/prepare_data.py from public sources; only the+# manifest is tracked.+datasets/*+!datasets/MANIFEST.jsondiff --git a/.gitmodules b/.gitmodulesnew file mode 100644index 0000000..fbfffdc--- /dev/null+++ b/.gitmodules@@ -0,0 +1,3 @@+[submodule "third_party/vllm"]+ path = third_party/vllm+ url = https://github.com/vllm-project/vllm.gitdiff --git a/README.md b/README.mdnew file mode 100644index 0000000..59d85d1--- /dev/null+++ b/README.md@@ -0,0 +1,49 @@+<p align="center">+ <img src="imgs/title.svg" alt="BitDecoding-FP4" width="550">+</p>++BitDecoding-FP4 is an accurate, high-performance **FP4 KV cache** for LLM+decoding, integrated into vLLM as an out-of-tree `CUSTOM` backend.++## Loss-less Accuracy++BitDecoding-NVFP4 achieves **near-lossless accuracy**: across seven benchmarks,+the average score stays within **0.5 percentage points of BF16 KV** on+Qwen3-4B-Thinking-2507 and Qwen3-30B-A3B-Thinking-2507.++++## Higher Serving Throughput.++On **NVIDIA B300**, BitDecoding-NVFP4 delivers **3.00×** (Qwen3-4B) and+**2.62×** (Qwen3-30B-A3B) the BF16 KV throughput:++++> Concurrency: 1, 8, 16, 32, 64.++## Setup++### Requirements++- 1 × NVIDIA B200 or B300 GPU (SM100 / SM103)+- CUDA 13.0 or newer (NVIDIA driver 580+)+- Python 3.12 and [uv](https://docs.astral.sh/uv/) (`pip install uv`)++### Install++```bash+git clone --recurse-submodules https://github.com/OpenBitSys/BitDecoding-FP4.git+cd BitDecoding-FP4+bash tools/bootstrap_env.sh+```++All the required packages are then installed in `_local/envs/bd_fp4`.++## Benchmark++See the [performance benchmark guide](benchmark/README.md) for kernel and e2e-serving options.++## Evaluation++See the [accuracy evaluation guide](evaluation/README.md) for dataset installation and the evaluation benchmarks used in the paper.diff --git a/benchmark/README.md b/benchmark/README.mdnew file mode 100644index 0000000..2096ae7--- /dev/null+++ b/benchmark/README.md@@ -0,0 +1,25 @@+# Benchmark++## Kernel benchmark++Compares BF16, TRT-LLM NVFP4, and BitDecoding NVFP4 decode latency using+synthetic inputs at 128K context:++```bash+REPO=/path/to/BitDecoding-FP4+CUDA_VISIBLE_DEVICES=0 "$REPO/_local/envs/bd_fp4/bin/python" "$REPO/benchmark/bench_kernel_sm10x.py"+```++> Defaults: Qwen3-4B and Qwen3-30B-A3B shapes at batch sizes 1, 16, 32, and 64; override with `--models` and `--batches`.++## E2E serving benchmark++Compares BF16, FP8, NVFP4, and BitDecoding NVFP4 decode throughput in vLLM at+1, 8, 16, 32, and 64 concurrent requests:++```bash+CUDA_VISIBLE_DEVICES=0 "$REPO/_local/envs/bd_fp4/bin/python" "$REPO/benchmark/bench_e2e_throughput.py" \+ --model Qwen/Qwen3-4B-Thinking-2507 # or Qwen/Qwen3-30B-A3B-Thinking-2507+```++> Defaults: 64K-token prompts on one GPU; for 128K on two GPUs, add `--workload 128k` and set `CUDA_VISIBLE_DEVICES=0,1`.diff --git a/benchmark/bench_e2e_throughput.py b/benchmark/bench_e2e_throughput.pynew file mode 100644index 0000000..f6a3ba7--- /dev/null+++ b/benchmark/bench_e2e_throughput.py@@ -0,0 +1,486 @@+#!/usr/bin/env python3+"""Reproduce the paper's 64K/TP1 or 128K/TP2 serving throughput sweep."""++from __future__ import annotations++import argparse+import copy+from collections import Counter+import csv+import hashlib+import importlib.metadata+import json+import math+import os+from pathlib import Path+import random+import statistics+import subprocess+import sys+import time++ROOT = Path(__file__).resolve().parents[1]+sys.path[:0] = [str(ROOT / "src"), str(ROOT), str(ROOT / "benchmark")]+METHODS = ("bf16", "fp8", "nvfp4", "bitdecoding-nvfp4")+WORKLOADS = {"64k": (65536, 1), "128k": (131072, 2)}+MODEL_REVISIONS = {+ "Qwen/Qwen3-4B-Thinking-2507": "768f209d9ea81521153ed38c47d515654e938aea",+ "Qwen/Qwen3-30B-A3B-Thinking-2507": "144afc2f379b542fdd4e85a1fcd5e1f79112d95d",+}+from evaluation.config import (+ RECIPE as ACCURACY_RECIPE,+ needs_tau_calibration,+ probability_environment,+ recipe_additional_config as custom_additional_config,+)++RECIPE = copy.deepcopy(ACCURACY_RECIPE)+++def digest(path: Path) -> str:+ with path.open("rb") as stream:+ return hashlib.file_digest(stream, "sha256").hexdigest()+++def save(path: Path, value: dict) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ temporary = path.with_suffix(path.suffix + ".tmp")+ temporary.write_text(json.dumps(value, indent=2) + "\n")+ temporary.replace(path)+++def make_prompts(tokenizer, corpus: list[dict], count: int, length: int) -> list[dict]:+ """Pack the same shuffled MRCR messages for both context lengths."""+ marker = "BODY_MARKER_45f24ec9"+ template = tokenizer.apply_chat_template(+ [{"role": "user", "content": marker}], tokenize=False,+ add_generation_prompt=True, enable_thinking=True,+ )+ prefix, suffix = template.split(marker)+ prefix = tokenizer.encode(prefix, add_special_tokens=False)+ suffix = tokenizer.encode(suffix, add_special_tokens=False)+ budget = 131072 - len(prefix) - len(suffix)+ if not corpus or length <= len(prefix) + len(suffix):+ raise ValueError("empty corpus or insufficient context for the chat template")+ order = list(range(len(corpus)))+ random.Random(42).shuffle(order)+ cache, prompts, cursor = {}, [], 0+ for index in range(count):+ body = []+ while len(body) < budget:+ source = order[cursor % len(order)]+ cursor += 1+ if source not in cache:+ text = "\n\n".join(+ message["role"] + ":\n" + message["content"]+ for message in corpus[source]["messages"]+ ) + "\n\n"+ cache[source] = tokenizer.encode(text, add_special_tokens=False)+ if not cache[source]:+ raise ValueError(f"corpus row {source} contains no text tokens")+ body.extend(cache[source])+ ids = prefix + body[:length - len(prefix) - len(suffix)] + suffix+ prompts.append({"prompt_token_ids": ids, "sampling_seed": 42 + index})+ return prompts+++def decode_rates(short: list[float], long: list[float], count: int,+ short_tokens: int, long_tokens: int, tp: int) -> dict:+ if (not short or len(short) != len(long) or count < 1 or tp < 1+ or not 0 < short_tokens < long_tokens+ or any(not math.isfinite(t) or t <= 0 for t in short + long)):+ raise ValueError("invalid paired timings or output lengths")+ elapsed = statistics.median(long) - statistics.median(short)+ differences = [b - a for a, b in zip(short, long, strict=True)]+ if elapsed <= 0 or any(value <= 0 for value in differences):+ raise ValueError("long generations must take longer than their paired baselines")+ per_request = (long_tokens - short_tokens) / elapsed+ return {+ "decode_seconds": elapsed,+ "tokens_per_second_per_request": per_request,+ "tokens_per_second_per_gpu": count * per_request / tp,+ "short_median_seconds": statistics.median(short),+ "long_median_seconds": statistics.median(long),+ "paired_differences_seconds": differences,+ }+++class ThroughputWorker:+ """Synchronize every TP rank without instrumenting the attention kernels."""++ def throughput_sync(self) -> None:+ import torch++ torch.cuda.synchronize()++ def throughput_state(self) -> dict:+ import torch+ from nvfp4_decode_kernel.p_scale import P_SMALL_GROUP_AMAX, resolve_p_scale_environment++ torch.cuda.synchronize()+ config = self.vllm_config+ model = self.get_model()+ layers = [module for module in model.modules()+ if hasattr(module, "attn_backend") and hasattr(module, "kv_cache_dtype")]+ return {+ "rank": self.rank,+ "gpu": torch.cuda.get_device_name(),+ "compute_capability": list(torch.cuda.get_device_capability()),+ "kv_cache_blocks": self.model_runner.kv_cache_config.num_blocks,+ "cache_dtypes": dict(Counter(str(layer.kv_cache_dtype) for layer in layers)),+ "backends": dict(Counter(layer.attn_backend.get_name() for layer in layers)),+ "peak_allocated_bytes": torch.cuda.max_memory_allocated(),+ "probability_scaling": {"mode": resolve_p_scale_environment()[0], "tau": P_SMALL_GROUP_AMAX},+ "engine": {+ "tp": config.parallel_config.tensor_parallel_size,+ "max_model_len": config.model_config.max_model_len,+ "max_num_seqs": config.scheduler_config.max_num_seqs,+ "block_size": config.cache_config.block_size,+ "gpu_memory_utilization": config.cache_config.gpu_memory_utilization,+ "enable_prefix_caching": config.cache_config.enable_prefix_caching,+ "enable_chunked_prefill": config.scheduler_config.enable_chunked_prefill,+ "enforce_eager": config.model_config.enforce_eager,+ "enable_flashinfer_autotune": config.kernel_config.enable_flashinfer_autotune,+ "cudagraph_mode": str(config.compilation_config.cudagraph_mode),+ },+ }+++class AdmissionObserver:+ """Count resident requests and preemptions without changing scheduling."""++ def __init__(self, llm):+ core = llm.llm_engine.engine_core+ while hasattr(core, "engine_core"):+ core = core.engine_core+ self.scheduler = core.scheduler+ self.original = self.scheduler.schedule+ self.reset()++ def reset(self) -> None:+ self.max_running = 0+ self.preemptions = {}++ def schedule(self, *args, **kwargs):+ result = self.original(*args, **kwargs)+ self.max_running = max(self.max_running, len(self.scheduler.running))+ for key, request in self.scheduler.requests.items():+ self.preemptions[key] = max(self.preemptions.get(key, 0), request.num_preemptions)+ return result++ def __enter__(self):+ self.scheduler.schedule = self.schedule+ return self++ def __exit__(self, *_):+ self.scheduler.schedule = self.original+++def measure(llm, rows: list[dict], observer, short_tokens: int,+ long_tokens: int, warmup: int, repeats: int, clock=time.perf_counter) -> dict:+ from vllm import SamplingParams, TokensPrompt++ prompts = [TokensPrompt(prompt_token_ids=row["prompt_token_ids"]) for row in rows]+ parameters = {+ length: [SamplingParams(+ temperature=0.6, top_p=0.95, top_k=20, min_p=0.0,+ min_tokens=length, max_tokens=length, ignore_eos=True,+ seed=row["sampling_seed"],+ ) for row in rows]+ for length in (short_tokens, long_tokens)+ }+ calls = []++ def run(length, measured):+ observer.reset()+ llm.collective_rpc("throughput_sync")+ start = clock()+ outputs = llm.generate(prompts, parameters[length], use_tqdm=False)+ llm.collective_rpc("throughput_sync")+ seconds = clock() - start+ if len(outputs) != len(rows):+ raise RuntimeError("not every submitted request completed")+ for row, output in zip(rows, outputs, strict=True):+ if (list(output.prompt_token_ids) != row["prompt_token_ids"]+ or len(output.outputs) != 1+ or len(output.outputs[0].token_ids) != length):+ raise RuntimeError("generated prompt or output length differs from the workload")+ calls.append({+ "output_tokens": length, "measured": measured, "seconds": seconds,+ "max_resident_requests": observer.max_running,+ "preemptions": sum(observer.preemptions.values()),+ })+ return seconds++ for _ in range(warmup):+ run(short_tokens, False)+ run(long_tokens, False)+ short, long = [], []+ for repeat in range(repeats):+ order = (short_tokens, long_tokens) if repeat % 2 == 0 else (long_tokens, short_tokens)+ times = {length: run(length, True) for length in order}+ short.append(times[short_tokens])+ long.append(times[long_tokens])+ return {"short_seconds": short, "long_seconds": long, "calls": calls}+++def engine_options(job: dict, *, calibration=False) -> dict:+ options = {+ "model": job["model"], "dtype": "bfloat16", "trust_remote_code": True,+ "tensor_parallel_size": 1 if calibration else job["tp"],+ "max_model_len": job["input_tokens"] + 2048,+ "max_num_seqs": 1 if calibration else job["concurrency"],+ "max_num_batched_tokens": job["input_tokens"] + 2048,+ "block_size": 128, "gpu_memory_utilization": 0.90,+ "enable_prefix_caching": False, "enable_chunked_prefill": False,+ "enforce_eager": False, "enable_flashinfer_autotune": False,+ }+ if not calibration:+ options["worker_extension_cls"] = "bench_e2e_throughput.ThroughputWorker"+ elif needs_tau_calibration(job["recipe"]):+ options["enforce_eager"] = True+ return options+++def build_engine(job: dict):+ options = engine_options(job)+ method = job["method"]+ if method == "bitdecoding-nvfp4":+ from nvfp4_vllm import build_llm++ recipe = job["recipe"]+ return build_llm(+ additional_config=custom_additional_config(+ recipe, key_smoothing=job["key_smoothing"]),+ **options,+ )+ options["kv_cache_dtype"] = {"bf16": "auto", "fp8": "fp8", "nvfp4": "nvfp4"}[method]+ from evaluation.vllm_runner import VllmRunner++ options["attention_config"] = {"backend": "FLASHINFER"}+ return VllmRunner._build_standard(+ options, {"kv_cache_dtype": options["kv_cache_dtype"]},+ worker_extension_cls=options["worker_extension_cls"],+ )+++def shutdown(llm) -> None:+ for owner in (llm, llm.llm_engine, llm.llm_engine.engine_core):+ close = getattr(owner, "shutdown", None)+ if callable(close):+ close()+ return+++def run_worker(job: dict) -> None:+ os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"+ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"+ recipe = job["recipe"]+ custom = job["kind"] == "calibrate" or job["method"] == "bitdecoding-nvfp4"+ environment = (probability_environment(recipe, calibration=job["kind"] == "calibrate")+ if custom else {"NVFP4_P_SCALE_MODE": None, "NVFP4_P_SCALE_TAU": None})+ for key, value in environment.items():+ if value is None:+ os.environ.pop(key, None)+ else:+ os.environ[key] = value+ os.environ.pop("NVFP4_P_SCALE_DIVISOR", None)+ if job["kind"] == "calibrate":+ from nvfp4_vllm import build_llm, calibrate_key_smoothing++ llm = build_llm(additional_config=custom_additional_config(recipe),+ **engine_options(job, calibration=True))+ try:+ multiplier = (recipe["underflow_aware_probability_scaling"]["c"]+ if needs_tau_calibration(recipe) else None)+ result = calibrate_key_smoothing(+ llm, context_tokens=job["input_tokens"] + 2047,+ save_to=job["key_smoothing"], probability_multiplier=multiplier,+ )+ if result is None or (job["key_smoothing"] and not Path(job["key_smoothing"]).is_file()):+ raise RuntimeError("key-smoothing calibration did not produce a sidecar")+ save(Path(job["result"]), result)+ finally:+ shutdown(llm)+ return++ rows = [json.loads(line) for line in Path(job["inputs"]).read_text().splitlines()]+ rows = rows[:job["concurrency"]]+ short_tokens, long_tokens = 128, 2048+ llm = build_engine(job)+ try:+ workers = llm.collective_rpc("throughput_state")+ expected_backend = "CUSTOM" if job["method"] == "bitdecoding-nvfp4" else "FLASHINFER"+ expected_dtype = {"bf16": "auto", "fp8": "fp8"}.get(job["method"], "nvfp4")+ layers = json.loads((Path(job["model"]) / "config.json").read_text())["num_hidden_layers"]+ if (len(workers) != job["tp"]+ or len({worker["rank"] for worker in workers}) != job["tp"]+ or any(worker["compute_capability"][0] != 10+ or worker["backends"] != {expected_backend: layers}+ or worker["cache_dtypes"] != {expected_dtype: layers} for worker in workers)):+ raise RuntimeError("GPU or loaded attention layers differ from the requested method")+ expected_engine = {+ "tp": job["tp"], "max_model_len": job["input_tokens"] + 2048,+ "max_num_seqs": job["concurrency"], "block_size": 128,+ "gpu_memory_utilization": 0.90, "enable_prefix_caching": False,+ "enable_chunked_prefill": False, "enforce_eager": False,+ "enable_flashinfer_autotune": False,+ }+ for worker in workers:+ if (any(worker["engine"][key] != value for key, value in expected_engine.items())+ or worker["engine"]["cudagraph_mode"] in ("None", "NONE", "CUDAGraphMode.NONE")):+ raise RuntimeError("resolved engine settings differ from the paper workload")+ if job["method"] == "bitdecoding-nvfp4":+ environment = probability_environment(recipe)+ if worker["probability_scaling"]["mode"] != environment["NVFP4_P_SCALE_MODE"]:+ raise RuntimeError("worker probability scaling differs from the recipe")+ if environment["NVFP4_P_SCALE_TAU"] is not None and (+ worker["probability_scaling"]["tau"] != float(environment["NVFP4_P_SCALE_TAU"])+ ):+ raise RuntimeError("worker did not load the calibrated tau")+ with AdmissionObserver(llm) as observer:+ timing = measure(llm, rows, observer, short_tokens, long_tokens,+ job["warmup"], job["repeats"])+ rates = decode_rates(timing["short_seconds"], timing["long_seconds"], len(rows),+ short_tokens, long_tokens, job["tp"])+ timed = [call for call in timing["calls"] if call["measured"]]+ resident = max(call["max_resident_requests"] for call in timed)+ preemptions = sum(call["preemptions"] for call in timed)+ save(Path(job["result"]), {+ **job, **rates, **timing, "baseline_tokens": short_tokens,+ "output_tokens": long_tokens, "max_resident_requests": resident,+ "preemptions": preemptions, "queued": resident < len(rows) or preemptions > 0,+ "workers": llm.collective_rpc("throughput_state"),+ "packages": {name: importlib.metadata.version(name) for name in+ ("torch", "vllm", "flashinfer-python", "nvidia-cutlass-dsl")},+ })+ finally:+ shutdown(llm)+++def launch(job: dict, directory: Path) -> None:+ job_path = directory / "job.json"+ save(job_path, job)+ environment = os.environ.copy()+ environment["PYTHONPATH"] = os.pathsep.join(+ [str(ROOT / "src"), str(ROOT), str(ROOT / "benchmark"), environment.get("PYTHONPATH", "")])+ with (directory / "run.log").open("w") as log:+ result = subprocess.run([sys.executable, "-u", str(Path(__file__).resolve()),+ "--worker", str(job_path)], env=environment,+ stdout=log, stderr=subprocess.STDOUT)+ if result.returncode:+ raise RuntimeError(f"benchmark failed; see {directory / 'run.log'}")+++def main() -> None:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--model", default="Qwen/Qwen3-4B-Thinking-2507",+ help="Qwen3 checkpoint directory or Hugging Face model ID")+ parser.add_argument("--workload", choices=WORKLOADS, default="64k")+ parser.add_argument("--methods", default=",".join(METHODS), help="comma-separated cache methods")+ parser.add_argument("--concurrency", default="1,8,16,32,64")+ parser.add_argument("--prompts", type=Path, help="MRCR messages JSONL; downloaded if omitted")+ parser.add_argument("--out", type=Path, help="empty output directory")+ parser.add_argument("--warmup", type=int, default=2)+ parser.add_argument("--repeats", type=int, default=3)+ parser.add_argument("--worker", type=Path, help=argparse.SUPPRESS)+ args = parser.parse_args()+ if args.worker:+ run_worker(json.loads(args.worker.read_text()))+ return+ methods = args.methods.split(",")+ try:+ concurrency = [int(value) for value in args.concurrency.split(",")]+ except ValueError:+ parser.error("--concurrency must contain comma-separated integers")+ if (not methods or set(methods) - set(METHODS) or len(methods) != len(set(methods))+ or not concurrency or any(c < 1 or c > 64 for c in concurrency)+ or len(concurrency) != len(set(concurrency)) or args.warmup < 0 or args.repeats < 1):+ parser.error(f"use distinct methods from {METHODS}, concurrency 1..64, and valid repeat counts")+ if args.prompts is not None and not args.prompts.is_file():+ parser.error(f"prompt corpus does not exist: {args.prompts}")+ name = args.model.rstrip("/").split("/")[-1]+ output = (args.out or ROOT / "_local/benchmark/e2e" / name / args.workload).resolve()+ if output.exists() and (not output.is_dir() or any(output.iterdir())):+ parser.error(f"output directory is not empty: {output}; choose another --out")+ model = Path(args.model).expanduser()+ if not model.is_dir():+ from huggingface_hub import snapshot_download++ model = Path(snapshot_download(args.model, revision=MODEL_REVISIONS.get(args.model)))+ model = model.resolve()+ config = json.loads((model / "config.json").read_text())+ if config.get("model_type") not in {"qwen3", "qwen3_moe"} or config.get("quantization_config"):+ parser.error("the paper workload requires an unquantized Qwen3 checkpoint")+ from transformers import AutoTokenizer++ tokenizer = AutoTokenizer.from_pretrained(str(model), trust_remote_code=True)+ corpus_path = args.prompts or ROOT / "datasets/mrcr/_filtered_le98048.jsonl"+ if not corpus_path.exists():+ from huggingface_hub import snapshot_download+ from tools.prepare_data import build_mrcr_cache++ snapshot_download("openai/mrcr", repo_type="dataset",+ local_dir=str(ROOT / "datasets/mrcr"),+ allow_patterns=[f"{n}needle/*.parquet" for n in (2, 4, 8)])+ build_mrcr_cache(str(model), budget=98048, out_path=str(corpus_path),+ manifest_key="datasets/mrcr/_filtered_le98048.jsonl")+ corpus = [json.loads(line) for line in corpus_path.read_text().splitlines() if line.strip()]+ length, tp = WORKLOADS[args.workload]+ prompts = make_prompts(tokenizer, corpus, max(concurrency), length)+ output.mkdir(parents=True, exist_ok=True)+ input_path = output / "prompts.jsonl"+ with input_path.open("w") as stream:+ for row in prompts:+ stream.write(json.dumps(row, separators=(",", ":")) + "\n")+ base = {+ "model": str(model), "workload": args.workload, "input_tokens": length, "tp": tp,+ "inputs": str(input_path), "input_sha256": digest(input_path),+ "corpus_sha256": digest(corpus_path), "warmup": args.warmup, "repeats": args.repeats,+ "recipe": copy.deepcopy(RECIPE),+ "key_smoothing": (str(output / "key_smoothing.safetensors")+ if RECIPE["outlier_aware_key_quantization"]["key_smoothing"] else None),+ }+ manifest = {**base, "methods": methods, "concurrency": concurrency,+ "requested_recipe": copy.deepcopy(RECIPE),+ "model_source": args.model,+ "model_revision": MODEL_REVISIONS.get(args.model),+ "model_config_sha256": digest(model / "config.json"),+ "benchmark_sha256": digest(Path(__file__)),+ "sampling": {"temperature": 0.6, "top_p": 0.95,+ "top_k": 20, "min_p": 0.0, "seed": "42 + index"}}+ save(output / "manifest.json", manifest)+ if "bitdecoding-nvfp4" in methods and (base["key_smoothing"] or needs_tau_calibration(RECIPE)):+ print("Calibrating key smoothing and probability scaling before timing...", flush=True)+ launch({**base, "kind": "calibrate", "result": str(output / "calibration.json")},+ output / "calibration")+ if needs_tau_calibration(RECIPE):+ calibration = json.loads((output / "calibration.json").read_text())+ tau = calibration["probability_scaling"]["tau"]+ base["recipe"]["underflow_aware_probability_scaling"]["tau"] = tau+ print(f"Calibrated tau={tau:.9g}", flush=True)+ save(output / "manifest.json", manifest)+ results = []+ for count in concurrency:+ for method in methods:+ directory = output / f"c{count}" / method+ result_path = directory / "result.json"+ print(f"{name} {args.workload} TP{tp} concurrency={count} {method}", flush=True)+ launch({**base, "kind": "measure", "method": method, "concurrency": count,+ "result": str(result_path)}, directory)+ result = json.loads(result_path.read_text())+ fields = ("method", "concurrency", "tp", "input_tokens", "output_tokens", "baseline_tokens",+ "tokens_per_second_per_gpu", "tokens_per_second_per_request", "decode_seconds",+ "max_resident_requests", "preemptions", "queued")+ results.append({field: result[field] for field in fields})+ with (output / "throughput.csv").open("w", newline="") as stream:+ writer = csv.DictWriter(stream, fieldnames=fields)+ writer.writeheader()+ writer.writerows(results)+ print(f" {result['tokens_per_second_per_gpu']:.2f} tokens/s/GPU, "+ f"{result['tokens_per_second_per_request']:.2f} tokens/s/request", flush=True)+ print(f"Results: {output / 'throughput.csv'}")+++if __name__ == "__main__":+ main()diff --git a/benchmark/bench_kernel_sm10x.py b/benchmark/bench_kernel_sm10x.pynew file mode 100644index 0000000..27b4408--- /dev/null+++ b/benchmark/bench_kernel_sm10x.py@@ -0,0 +1,386 @@+#!/usr/bin/env python3+"""Decode attention kernel speedup over BF16 KV at long context on SM100 / SM103.++- BF16 KV: FlashInfer trtllm-gen BF16 decode, what vLLM runs for BF16 decode+ on SM100.+- TRT-LLM-NVFP4: FlashInfer trtllm-gen NVFP4 decode with an FP8 query.+- BitDecoding-NVFP4: this repository's ``fp4_decode`` with a two-term FP4+ query over FP4 pages, at the fastest of up to four candidate split counts.++ python benchmark/bench_kernel_sm10x.py+ python benchmark/bench_kernel_sm10x.py --models qwen3-4b --batches 1,64+"""++from __future__ import annotations++import argparse+import gc+import importlib.metadata+import json+import statistics+import subprocess+import sys+from datetime import datetime, timezone+from pathlib import Path++import torch++ROOT = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(ROOT / "src"))++from flashinfer.decode import trtllm_batch_decode_with_kv_cache # noqa: E402+from flashinfer.fp4_quantization import nvfp4_kv_quantize # noqa: E402+from nvfp4_decode_kernel import _quantize, fp4_decode, split_partial_count # noqa: E402+from nvfp4_decode_kernel._decode import split_k_heuristic # noqa: E402+from nvfp4_vllm.builder import decode_num_splits # noqa: E402++PAGE_SIZE = 128+HEAD_DIM = 128+HEADS_Q = 32+# name -> (title, KV heads)+MODELS = {"qwen3-4b": ("Qwen3-4B", 8), "qwen3-30b-a3b": ("Qwen3-30B-A3B", 4)}+SIDES = {+ "bf16": "BF16 KV",+ "trtllm_nvfp4": "TRT-LLM-NVFP4",+ "bitdecoding_nvfp4": "BitDecoding-NVFP4",+}+FP8 = torch.float8_e4m3fn+E4M3_MAX = 448.0+# vLLM's FlashInfer workspace size.+WORKSPACE_BYTES = 394 * 1024 * 1024+# Bound temporary allocations while preparing the benchmark's quantized cache.+QUANTIZE_CHUNK_PAGES = 32768+TRTLLM_CHUNK_PAGES = 512+++def amax(x: torch.Tensor) -> float:+ lo, hi = torch.aminmax(x)+ return max(-lo.item(), hi.item())+++def cosine(a: torch.Tensor, b: torch.Tensor) -> float:+ a, b = a.float().flatten(), b.float().flatten()+ return float((a @ b) / (a.norm() * b.norm() + 1e-30))+++def quantize_pages(quantizer, pages: torch.Tensor):+ """This repository's FP4 pages and scales, one launch per chunk of pages."""+ count = pages.shape[0]+ data = scales = None+ for start in range(0, count, QUANTIZE_CHUNK_PAGES):+ stop = min(start + QUANTIZE_CHUNK_PAGES, count)+ chunk_data, chunk_scales = quantizer(pages[start:stop])+ if data is None:+ data = torch.empty((count, *chunk_data.shape[1:]), dtype=chunk_data.dtype,+ device=pages.device)+ # The scales are a strided view in the kernel's swizzled layout;+ # only the page stride changes with the page count.+ storage = torch.empty(count * chunk_scales.stride(0), dtype=torch.uint8,+ device=pages.device)+ scales = storage.as_strided((count, *chunk_scales.shape[1:]),+ chunk_scales.stride()).view(chunk_scales.dtype)+ data[start:stop].copy_(chunk_data)+ scales[start:stop].copy_(chunk_scales)+ return data, scales+++def trtllm_nvfp4_pages(pages: torch.Tensor, heads_kv: int):+ """trtllm-gen NVFP4 pages: HND layout, linear E4M3 scales, a global descale.++ trtllm-gen dequantizes to FP8 before its MMAs, so a block's ``e2m1 * scale``+ has to fit in E4M3; a descale of ``amax / 448`` keeps block scales at or+ below 448 / 6.+ """+ num_pages = pages.shape[0]+ descale = amax(pages) / E4M3_MAX+ global_scale = torch.tensor([descale], dtype=torch.float32, device=pages.device)+ data = torch.empty(num_pages, heads_kv, PAGE_SIZE, HEAD_DIM // 2, dtype=torch.uint8,+ device=pages.device)+ scales = torch.empty(num_pages, heads_kv, PAGE_SIZE, HEAD_DIM // 16, dtype=FP8,+ device=pages.device)+ for start in range(0, num_pages, TRTLLM_CHUNK_PAGES):+ stop = min(start + TRTLLM_CHUNK_PAGES, num_pages)+ block = pages[start:stop].permute(0, 2, 1, 3).contiguous()+ block_data, block_scales = nvfp4_kv_quantize(block.reshape(-1, HEAD_DIM), global_scale)+ data[start:stop].copy_(block_data.view(-1, heads_kv, PAGE_SIZE, HEAD_DIM // 2))+ scales[start:stop].copy_(+ block_scales.view(-1, heads_kv, PAGE_SIZE, HEAD_DIM // 16).view(FP8))+ return data, scales, descale+++def split_candidates(batch: int, heads_kv: int, seqlen: int, sms: int) -> list[int]:+ tiles, pages = batch * heads_kv, seqlen // PAGE_SIZE+ candidates = {+ decode_num_splits(batch, heads_kv, sms), # what the vLLM backend picks+ split_k_heuristic(batch, heads_kv, pages, sms=sms),+ max(1, sms // tiles), # one wave of CTAs+ }+ # The fewest splits whose last wave of CTAs is at least 97% full, keeping+ # at least 64 pages per split.+ for splits in range(1, 65):+ if pages // splits < 64:+ break+ ctas = tiles * splits+ if ctas / (-(-ctas // sms) * sms) >= 0.97:+ candidates.add(splits)+ break+ return sorted(candidates)+++def build_sides(batch: int, heads_kv: int, seqlen: int, splits: list[int], sms: int,+ device) -> dict:+ """name -> (zero-argument kernel call, its output as FP32 for the cosine check)."""+ pages_per_row = seqlen // PAGE_SIZE+ num_pages = batch * pages_per_row+ softmax_scale = HEAD_DIM**-0.5+ query = torch.randn(batch, HEADS_Q, HEAD_DIM, dtype=torch.bfloat16, device=device).mul_(0.3)+ k = torch.randn(num_pages, PAGE_SIZE, heads_kv, HEAD_DIM, dtype=torch.bfloat16,+ device=device).mul_(0.3)+ v = torch.randn_like(k).mul_(0.3)+ page_table = torch.arange(num_pages, dtype=torch.int32, device=device).reshape(+ batch, pages_per_row)+ seq_lens = torch.full((batch,), seqlen, dtype=torch.int32, device=device)++ k_fp4, k_scales = quantize_pages(_quantize.quantize_key_pages, k)+ v_fp4, v_scales = quantize_pages(_quantize.quantize_value_pages, v)+ q0, s0, q1, s1 = _quantize.quantize_query(query, heads_kv=heads_kv, query_terms=2)++ k_trt, k_trt_scales, k_descale = trtllm_nvfp4_pages(k, heads_kv)+ v_trt, v_trt_scales, v_descale = trtllm_nvfp4_pages(v, heads_kv)+ q_descale = amax(query) / E4M3_MAX+ query_fp8 = (query.float() / q_descale).to(FP8)+ # trtllm-gen NVFP4 only writes E4M3. Attention output is a convex+ # combination of V rows, so |out| <= amax(V) and this descale cannot saturate.+ out_descale = amax(v) / E4M3_MAX++ k_hnd = k.permute(0, 2, 1, 3).contiguous()+ del k+ v_hnd = v.permute(0, 2, 1, 3).contiguous()+ del v++ def trtllm(query, kv_cache, out, **scales):+ workspace = torch.zeros(WORKSPACE_BYTES, dtype=torch.uint8, device=device)++ def run():+ trtllm_batch_decode_with_kv_cache(+ query=query, kv_cache=kv_cache, workspace_buffer=workspace,+ block_tables=page_table, seq_lens=seq_lens, max_seq_len=seqlen, out=out,+ kv_layout="HND", backend="trtllm-gen", q_len_per_req=1, **scales)+ return out++ return run++ bf16_out = torch.empty(batch, HEADS_Q, HEAD_DIM, dtype=torch.bfloat16, device=device)+ trt_out = torch.empty(batch, HEADS_Q, HEAD_DIM, dtype=FP8, device=device)+ sides = {+ "bf16": (+ trtllm(query, (k_hnd, v_hnd), bf16_out,+ bmm1_scale=softmax_scale, bmm2_scale=1.0),+ lambda out: out.float(),+ ),+ "trtllm_nvfp4": (+ trtllm(query_fp8, (k_trt, v_trt), trt_out,+ bmm1_scale=q_descale * k_descale * softmax_scale,+ bmm2_scale=v_descale / out_descale,+ kv_cache_sf=(k_trt_scales, v_trt_scales)),+ lambda out: out.float() * out_descale,+ ),+ }++ def bitdecoding(num_splits: int):+ out = torch.empty(batch, HEADS_Q, HEAD_DIM, dtype=torch.bfloat16, device=device)+ scratch = {}+ if num_splits > 1:+ partials = split_partial_count(batch, heads_kv, num_splits, residual=False, sms=sms)+ scratch = {+ "split_partial_scratch": torch.empty(+ partials * batch * HEADS_Q * HEAD_DIM, dtype=torch.float32, device=device),+ "split_lse_scratch": torch.empty(+ partials * batch * HEADS_Q, dtype=torch.float32, device=device),+ }++ def run():+ return fp4_decode(+ key_pages_fp4=k_fp4, key_scales=k_scales,+ value_pages_fp4=v_fp4, value_scales=v_scales,+ fp4_page_table=page_table, seqused_fp4=seq_lens,+ query_fp4=q0, query_scales=s0, query_fp4_second=q1, query_scales_second=s1,+ query_terms=2, softmax_scale=softmax_scale, trusted_metadata=True,+ num_splits=num_splits, out=out, **scratch)++ return run++ for num_splits in splits:+ sides[f"bitdecoding_nvfp4_s{num_splits}"] = (bitdecoding(num_splits), lambda out: out.float())+ return sides+++def capture(run, calls: int) -> torch.cuda.CUDAGraph:+ stream = torch.cuda.Stream()+ stream.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(stream):+ for _ in range(3):+ run()+ torch.cuda.current_stream().wait_stream(stream)+ torch.cuda.synchronize()+ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph):+ for _ in range(calls):+ run()+ return graph+++def time_us(graph: torch.cuda.CUDAGraph, calls: int, args) -> float:+ """Median microseconds per call over ``args.repeats`` timed batches of replays."""+ for _ in range(args.warmup):+ graph.replay()+ torch.cuda.synchronize()+ samples = []+ for _ in range(args.repeats):+ start = torch.cuda.Event(enable_timing=True)+ end = torch.cuda.Event(enable_timing=True)+ start.record()+ for _ in range(args.iters):+ graph.replay()+ end.record()+ end.synchronize()+ samples.append(start.elapsed_time(end) * 1e3 / (args.iters * calls))+ return statistics.median(samples)+++def run_case(model: str, batch: int, args, device, sms: int) -> dict:+ title, heads_kv = MODELS[model]+ torch.manual_seed(args.seed + batch)+ torch.cuda.reset_peak_memory_stats(device)+ splits = split_candidates(batch, heads_kv, args.seqlen, sms)+ sides = build_sides(batch, heads_kv, args.seqlen, splits, sms, device)++ # The first call compiles each kernel and is the accuracy sample.+ outputs = {}+ for name, (run, as_float) in sides.items():+ outputs[name] = as_float(run()).clone()+ torch.cuda.synchronize()+ cosines = {name: cosine(out, outputs["bf16"]) for name, out in outputs.items() if name != "bf16"}++ graphs = {name: capture(run, args.calls_per_graph) for name, (run, _) in sides.items()}+ names = list(graphs)+ samples = {name: [] for name in names}+ for round_index in range(args.rounds):+ for name in names if round_index % 2 == 0 else names[::-1]:+ samples[name].append(time_us(graphs[name], args.calls_per_graph, args))+ measured = {name: statistics.median(values) for name, values in samples.items()}++ by_split = {s: measured[f"bitdecoding_nvfp4_s{s}"] for s in splits}+ best_split = min(by_split, key=by_split.get)+ us = {+ "bf16": measured["bf16"],+ "trtllm_nvfp4": measured["trtllm_nvfp4"],+ "bitdecoding_nvfp4": by_split[best_split],+ }+ row = {+ "model": model, "title": title, "batch": batch, "seqlen": args.seqlen,+ "heads_q": HEADS_Q, "heads_kv": heads_kv,+ "us": us,+ "speedup": {side: us["bf16"] / value for side, value in us.items()},+ "bitdecoding_splits": best_split,+ "bitdecoding_us_by_splits": by_split,+ "cosine_vs_bf16": {+ "trtllm_nvfp4": cosines["trtllm_nvfp4"],+ "bitdecoding_nvfp4": cosines[f"bitdecoding_nvfp4_s{best_split}"],+ },+ "samples_us": samples,+ "peak_memory_gib": torch.cuda.max_memory_allocated(device) / 2**30,+ }+ print(f"{title:<14} batch {batch:>3} | BF16 KV {us['bf16']:8.1f} us"+ f" | TRT-LLM-NVFP4 {us['trtllm_nvfp4']:8.1f} us ({row['speedup']['trtllm_nvfp4']:.2f}x)"+ f" | BitDecoding-NVFP4 {us['bitdecoding_nvfp4']:8.1f} us"+ f" ({row['speedup']['bitdecoding_nvfp4']:.2f}x, {best_split} splits)", flush=True)+ return row+++def summary(rows: list[dict]) -> str:+ lines = ["", "Speedup over BF16 KV"]+ for model in dict.fromkeys(row["model"] for row in rows):+ model_rows = [row for row in rows if row["model"] == model]+ lines.append(f" {MODELS[model][0]}")+ lines.append(" " + f"{'batch':<20}" + "".join(f"{row['batch']:>8}" for row in model_rows))+ for side in ("trtllm_nvfp4", "bitdecoding_nvfp4"):+ lines.append(" " + f"{SIDES[side]:<20}"+ + "".join(f"{row['speedup'][side]:>8.2f}" for row in model_rows))+ return "\n".join(lines)+++def provenance(device, args) -> dict:+ def version(name: str) -> str | None:+ try:+ return importlib.metadata.version(name)+ except importlib.metadata.PackageNotFoundError:+ return None++ try:+ commit = subprocess.run(["git", "-C", str(ROOT), "rev-parse", "HEAD"],+ capture_output=True, text=True, check=True).stdout.strip()+ except (OSError, subprocess.CalledProcessError):+ commit = None+ props = torch.cuda.get_device_properties(device)+ return {+ "timestamp_utc": datetime.now(timezone.utc).isoformat(),+ "commit": commit,+ "gpu": props.name,+ "compute_capability": f"{props.major}.{props.minor}",+ "sms": props.multi_processor_count,+ "packages": {name: version(name) for name in+ ("torch", "flashinfer-python", "nvidia-cutlass-dsl")},+ "arguments": {key: str(value) if isinstance(value, Path) else value+ for key, value in vars(args).items()},+ }+++def main() -> None:+ parser = argparse.ArgumentParser(description=__doc__,+ formatter_class=argparse.RawDescriptionHelpFormatter)+ parser.add_argument("--models", default=",".join(MODELS),+ help=f"comma-separated, from {', '.join(MODELS)}")+ parser.add_argument("--batches", default="1,16,32,64")+ parser.add_argument("--seqlen", type=int, default=131072,+ help="context length per row, a multiple of 128")+ parser.add_argument("--rounds", type=int, default=3)+ parser.add_argument("--warmup", type=int, default=30, help="graph replays before timing")+ parser.add_argument("--iters", type=int, default=20, help="graph replays per timed sample")+ parser.add_argument("--repeats", type=int, default=5, help="timed samples per round")+ parser.add_argument("--calls-per-graph", type=int, default=10)+ parser.add_argument("--seed", type=int, default=0)+ parser.add_argument("--out", type=Path, default=ROOT / "_local/benchmark/kernel_sm10x.json")+ args = parser.parse_args()++ models = args.models.split(",")+ unknown = [model for model in models if model not in MODELS]+ if unknown:+ parser.error(f"unknown model(s) {unknown}; choose from {list(MODELS)}")+ if args.seqlen % PAGE_SIZE:+ parser.error("--seqlen must be a multiple of 128")+ batches = [int(value) for value in args.batches.split(",")]++ device = torch.device("cuda:0")+ torch.cuda.set_device(device)+ major, minor = torch.cuda.get_device_capability(device)+ if major != 10:+ raise SystemExit(f"needs an SM100-family GPU (B200 / B300); "+ f"{torch.cuda.get_device_name(device)} is sm_{major}{minor}")+ sms = torch.cuda.get_device_properties(device).multi_processor_count++ result = {**provenance(device, args), "rows": []}+ args.out.parent.mkdir(parents=True, exist_ok=True)+ for model in models:+ for batch in batches:+ result["rows"].append(run_case(model, batch, args, device, sms))+ args.out.write_text(json.dumps(result, indent=2) + "\n")+ gc.collect()+ torch.cuda.empty_cache()++ print(summary(result["rows"]))+ print(f"\nwrote {args.out}")+++if __name__ == "__main__":+ main()diff --git a/datasets/MANIFEST.json b/datasets/MANIFEST.jsonnew file mode 100644index 0000000..1ef49db--- /dev/null+++ b/datasets/MANIFEST.json@@ -0,0 +1,199 @@+{+ "schema": 1,+ "description": "Provenance for every public dataset BitDecoding-FP4 reads. tools/prepare_data.py stages upstreams and rebuilds derived artifacts, then checks them against the metadata below.",+ "upstream": {+ "caskcsg/LongBench-Pro": {+ "kind": "hf_dataset",+ "revision": "4996884deae51f5e5d23c88da9d857fc54e5fa15",+ "target": "datasets/longbench_pro",+ "format": "datasets.save_to_disk",+ "used_by": "eval/longbench_pro.py",+ "license": "apache-2.0",+ "paper": "https://arxiv.org/abs/2601.02872"+ },+ "openai/mrcr": {+ "kind": "hf_dataset",+ "files": [+ "2needle/*.parquet",+ "4needle/*.parquet",+ "8needle/*.parquet"+ ],+ "target": "datasets/mrcr",+ "approx_bytes": 1386000000,+ "used_by": "the derived MRCR caches below",+ "license": "mit",+ "paper": "https://arxiv.org/abs/2409.12640"+ },+ "HuggingFaceH4/aime_2024": {+ "kind": "hf_dataset",+ "revision": "2fe88a2f1091d5048c0f36abc874fb997b3dd99a",+ "target": "datasets/aime24/aime24.parquet",+ "n_rows": 30,+ "used_by": "eval/aime.py (30 problems; Avg@32 repeats are made at load time)",+ "license": "mit"+ },+ "math-ai/aime25": {+ "kind": "hf_dataset",+ "revision": "563bb8404243c5f09de6ec262f2db674fe5bce9b",+ "target": "datasets/aime25/aime25.parquet",+ "n_rows": 30,+ "used_by": "eval/aime25.py (30 problems; Avg@32 repeats are made at load time)"+ },+ "livecodebench/code_generation_lite": {+ "kind": "hf_file",+ "file": "test6.jsonl",+ "target": "datasets/lcb_v6/prompts.jsonl",+ "used_by": "eval/lcb.py (prompts only; tests stay in the LiveCodeBench checkout)"+ },+ "Qwen/Qwen3-4B-Thinking-2507": {+ "kind": "hf_model",+ "revision": "768f209d9ea81521153ed38c47d515654e938aea",+ "target": "_local/models/Qwen3-4B-Thinking-2507",+ "native_context_tokens": 262144,+ "config_sha256": "5beea1a4a34c62782bfb2f911c606741a3bab8f92d80a118fa053c28af12e8ba",+ "model_index_sha256": "d6c42883a895dfef5b0080ed2116a1bcd764f558406b98923d675978a1abf29c",+ "tokenizer_vocab_sha256": "6e949d8fec10c504f4207a43cdeac7d7961f7fd9894d256f14a8153bb5479b63",+ "staging": "tools/prepare_data.py --steps long-context-model",+ "used_by": "long-context KV-cache evaluation",+ "license": "apache-2.0"+ },+ "NVIDIA/RULER": {+ "kind": "github_source",+ "revision": "c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a",+ "target": "datasets/ruler",+ "used_by": "eval/ruler.py and tools/prepare_data.py --steps ruler",+ "license": "apache-2.0",+ "paper": "https://arxiv.org/abs/2404.06654"+ },+ "openai/simple-evals": {+ "kind": "github_file",+ "url": "https://raw.githubusercontent.com/openai/simple-evals/main/data/gpqa_diamond.csv",+ "target": "datasets/gpqa/gpqa_diamond.parquet",+ "n_rows": 198,+ "used_by": "eval/gpqa.py"+ }+ },+ "derived": {+ "datasets/longbench_pro": {+ "from": [+ "caskcsg/LongBench-Pro"+ ],+ "rebuild": "tools/prepare_data.py --steps longbench-pro",+ "n_rows": 1500,+ "n_rows_by_bucket": {+ "8k": 250,+ "16k": 250,+ "32k": 250,+ "64k": 250,+ "128k": 250,+ "256k": 250+ },+ "content_sha256": "2c92f9e1d4b0c9473f07845e3ef4ddebed66e588da737c5a4aababfa8729bca6"+ },+ "datasets/aime24/aime24.parquet": {+ "from": [+ "HuggingFaceH4/aime_2024"+ ],+ "rebuild": "tools/prepare_data.py --steps aime24",+ "n_rows": 30,+ "bytes": 10258,+ "sha256": "6baa662e42694fd7bc816c1c04f3b5db53a47710311baed44aab2f14d9eaff27",+ "content_sha256": "9bccf152c547eacb67e188b26289f69237ec1356b7dbd0d0c5dbf859a806fae9"+ },+ "datasets/aime25/aime25.parquet": {+ "from": [+ "math-ai/aime25"+ ],+ "rebuild": "tools/prepare_data.py --steps aime25",+ "n_rows": 30,+ "bytes": 14312,+ "sha256": "bfb7ed2e1546f341970901551c22da990bf6c3faf238859f2a4d53af97a48178",+ "content_sha256": "80ab018e2b800e20f763566d197b5f07fe11a01e8bb9dcde1266014693ece3b1"+ },+ "datasets/mrcr/_filtered_le34000.jsonl": {+ "from": [+ "openai/mrcr"+ ],+ "rebuild": "tools/prepare_data.py --steps mrcr",+ "rule": "Walk n_needles groups 2, 4, 8 in shard order; keep a row when sum(len(tokenizer(m['content'], add_special_tokens=False)) for m in json.loads(prompt)) <= 34000 with the Qwen3 tokenizer. Chat-template markup is deliberately not counted.",+ "max_prompt_tokens": 34000,+ "tokenizer": "Qwen/Qwen3-8B",+ "n_rows": 913,+ "n_rows_by_needles": {+ "2": 304,+ "4": 305,+ "8": 304+ },+ "bytes": 67097346,+ "sha256": "a7f659380654740dfb6f5c2151177331146933ade5fbe1af3892fc9d2a4ce876"+ },+ "datasets/mrcr/_filtered_le229120.jsonl": {+ "from": [+ "openai/mrcr",+ "Qwen/Qwen3-4B-Thinking-2507"+ ],+ "rebuild": "tools/prepare_data.py --steps long-context-mrcr",+ "max_prompt_tokens": 229120,+ "tokenizer": "Qwen/Qwen3-4B-Thinking-2507",+ "n_rows": 1724,+ "bytes": 490601488,+ "sha256": "dfbd2a0fe13e35011fddf1e7198bdc8543212b4d61f67f0ee6eaff20f01938b4"+ },+ "datasets/mrcr/_filtered_le98048.jsonl": {+ "from": [+ "openai/mrcr",+ "Qwen/Qwen3-4B-Thinking-2507"+ ],+ "rebuild": "tools/prepare_data.py --steps long-context-mrcr --long-context-mrcr-budget 98048 --force",+ "max_prompt_tokens": 98048,+ "tokenizer": "Qwen/Qwen3-4B-Thinking-2507",+ "n_rows": 1370,+ "bytes": 210660975,+ "sha256": "8d8d452993c4703a3b0eb7a7d8677ca065fe455beeff4bee06c12449717688e0"+ },+ "datasets/ruler/niah_122880.jsonl": {+ "from": [+ "NVIDIA/RULER",+ "Qwen/Qwen3-4B-Thinking-2507"+ ],+ "rebuild": "tools/prepare_data.py --steps ruler --force",+ "generator_revision": "c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a",+ "random_seed": 42,+ "max_seq_length": 122880,+ "num_samples_per_task": 500,+ "n_rows": 4000,+ "n_rows_by_task": {+ "niah_single_1": 500,+ "niah_single_2": 500,+ "niah_single_3": 500,+ "niah_multikey_1": 500,+ "niah_multikey_2": 500,+ "niah_multikey_3": 500,+ "niah_multivalue": 500,+ "niah_multiquery": 500+ },+ "tokenizer_vocab_sha256": "6e949d8fec10c504f4207a43cdeac7d7961f7fd9894d256f14a8153bb5479b63",+ "sha256": "6ab64b49cbadd9eefdb8621ad3ff5c3afd6ac32017cf67e4eefccaaf0b2201dc"+ },+ "datasets/ruler/generator.json": {+ "from": [+ "NVIDIA/RULER",+ "Qwen/Qwen3-4B-Thinking-2507"+ ],+ "rebuild": "tools/prepare_data.py --steps ruler --force",+ "note": "Installation paths vary by host; each generated file and shard is verified using its recorded row count and SHA-256."+ },+ "datasets/gpqa/gpqa_diamond.parquet": {+ "from": [+ "openai/simple-evals"+ ],+ "n_rows": 198+ },+ "datasets/lcb_v6/prompts.jsonl": {+ "from": [+ "livecodebench/code_generation_lite"+ ],+ "n_rows": 175+ }+ }+}diff --git a/evaluation/README.md b/evaluation/README.mdnew file mode 100644index 0000000..ed20478--- /dev/null+++ b/evaluation/README.md@@ -0,0 +1,34 @@+# Accuracy evaluation++Evaluates LongBench-Pro, MRCR, RULER-NIAH, AIME 2024/2025, LiveCodeBench v6, and GPQA-Diamond.++## Datasets++```bash+cd /path/to/BitDecoding-FP4+PY="$PWD/_local/envs/bd_fp4/bin/python"+export MODEL_PATH=/path/to/Qwen3-4B-Thinking-2507+"$PY" tools/prepare_data.py --steps eval --tokenizer "$MODEL_PATH"+```++Downloads the datasets into `datasets/` and prepares RULER and LiveCodeBench under `_local/`.++## Run++Evaluate at 128K context:++```bash+CUDA_VISIBLE_DEVICES=0 "$PY" -m evaluation.run_eval --model-path "$MODEL_PATH" \+ --modes bitdecoding-nvfp4 --thinking --out-dir _local/accuracy+```++* Edit `EVALUATION` in [config.py](config.py) for context length, batch size, and dataset settings++* edit `RECIPE` for the paper's three methods. Key smoothing and τ are calibrated together and saved under `_local/accuracy/calibration/`.+++LiveCodeBench scoring runs separately:++```bash+"$PY" tools/score_lcb.py --run-dir _local/accuracy --mode bitdecoding-nvfp4+```diff --git a/evaluation/__init__.py b/evaluation/__init__.pynew file mode 100644index 0000000..c6bcee9--- /dev/null+++ b/evaluation/__init__.py@@ -0,0 +1 @@+"""BitDecoding-FP4 evaluation harness (NVFP4 KV-cache quality)."""diff --git a/evaluation/aime.py b/evaluation/aime.pynew file mode 100644index 0000000..e78780a--- /dev/null+++ b/evaluation/aime.py@@ -0,0 +1,34 @@+"""AIME 2024 accuracy at Avg@32, scored on the last boxed integer.++This is the official contest metric (integer 0--999, exact match) plus the+Qwen3 eval protocol: ``Please reason step by step, and put your final answer+within \\boxed{}``, thinking-mode sampling, Avg@32. The grader is the same as+:mod:`evaluation.aime25`. Repeats are made at load time from the 30-problem file;+they are a sampling decision, not data.++DAPO's Minerva ``Answer:`` scorer is a training reward and is not used here.+"""+from __future__ import annotations++from . import aime25, config++METRIC_KEY = aime25.METRIC_KEY+REPEATS = aime25.REPEATS+last_boxed = aime25.last_boxed+as_integer = aime25.as_integer+build = aime25.build+seeds = aime25.seeds+grade = aime25.grade+score_jsonl = aime25.score_jsonl+++def load_examples(path: str = None):+ return aime25.load_examples(path or config.AIME24_PARQUET)+++def main() -> None:+ aime25.main()+++if __name__ == "__main__":+ main()diff --git a/evaluation/aime25.py b/evaluation/aime25.pynew file mode 100644index 0000000..51f971f--- /dev/null+++ b/evaluation/aime25.py@@ -0,0 +1,214 @@+"""AIME 2025 accuracy at Avg@32, scored on the boxed integer.++The staged file holds the 30 problems once; the 32 repeats are made here.+Repeats are a sampling decision, not data, so they are not written to disk.+:mod:`evaluation.aime` uses the same grader on the 2024 set.++Temperature is 0.6, so the 32 repeats of a problem must not share a seed or they+would decode identically and Avg@32 would collapse to Avg@1. :func:`seeds`+derives one per row from the problem and repeat number rather than from the+row's position, so it survives sharding and is the same in every arm -- two arms+then differ by their KV cache and nothing else.++Grading is exact match on the last ``\\boxed{}``, read as an integer. AIME+answers are integers 0--999, so this is not an approximation of symbolic+verification but an equivalent of it on this answer space, and it needs no+dependency that can silently change its mind between runs. ``math_verify``+still runs when it is installed, and its verdict is recorded alongside rather+than instead: the two disagreeing is a fact about the extraction worth seeing,+and the report states the rate.+"""+from __future__ import annotations++import json+import re+import sys+from typing import List++from . import config+++METRIC_KEY = "acc"++# Avg@32. One request per row; the repeats live in the example list.+REPEATS = 32++# Qwen's own instruction for math, from the Qwen3-8B model card. The answer+# extractor below depends on it, so the two move together.+ANSWER_INSTRUCTION = (+ "Please reason step by step, and put your final answer within \\boxed{}."+)+++def load_examples(path: str = None) -> List[dict]:+ import pandas as pd++ frame = pd.read_parquet(path or config.AIME25_PARQUET)+ examples = []+ for problem, record in enumerate(frame.to_dict(orient="records")):+ content = f"{record['problem']}\n\n{ANSWER_INSTRUCTION}"+ for repeat in range(REPEATS):+ examples.append({+ "problem": problem,+ "repeat": repeat,+ "messages": [{"role": "user", "content": content}],+ "ground_truth": str(record["answer"]),+ })+ return examples+++def build(examples):+ return [ex["messages"] for ex in examples], [{"example": ex} for ex in examples]+++def seeds(prepared, base_seed: int) -> List[int]:+ """One sampling seed per row, keyed by what the row *is*.++ Not by where it sits: shards are strided, so a position-derived seed would+ give a row a different one at a different shard count, and the arms would+ stop being comparable to anything run earlier.+ """+ return [+ base_seed + 1_000_003 * item["example"]["problem"] + item["example"]["repeat"]+ for item in prepared+ ]+++def last_boxed(text: str) -> str | None:+ """The content of the final ``\\boxed{...}``, brace-balanced.++ The final one rather than the first because a thinking model boxes its+ working as it goes; the answer is the last thing it boxes. Balanced rather+ than a regex because the content can itself contain braces.+ """+ start = text.rfind("\\boxed")+ if start < 0:+ return None+ open_brace = text.find("{", start)+ if open_brace < 0:+ return None+ depth = 0+ for index in range(open_brace, len(text)):+ if text[index] == "{":+ depth += 1+ elif text[index] == "}":+ depth -= 1+ if depth == 0:+ return text[open_brace + 1:index]+ return None+++def as_integer(boxed: str | None) -> int | None:+ """An AIME answer, or None if what was boxed is not one."""+ if boxed is None:+ return None+ cleaned = boxed.strip()+ for junk in ("\\!", "\\,", "\\;", "\\ ", "$", ",", " ", "\\left", "\\right"):+ cleaned = cleaned.replace(junk, "")+ cleaned = re.sub(r"^\\text\{(.*)\}$", r"\1", cleaned).strip()+ if not re.fullmatch(r"[+-]?\d+(?:\.0+)?", cleaned):+ return None+ try:+ return int(cleaned.split(".", 1)[0])+ except ValueError:+ # Invalid model output can exceed Python's integer-string limit.+ return None+++def _math_verify(prediction: str, ground_truth: str):+ """``math_verify``'s verdict, or None if it is unavailable or fails.++ Never allowed to decide the metric, and never allowed to end a run: an+ optional cross-check that raises is a cross-check that stops the+ evaluation it was added to observe.+ """+ try:+ from math_verify import parse, verify+ except ImportError:+ return None+ try:+ gold = parse(f"${ground_truth}$")+ answer = parse(prediction)+ return float(bool(verify(gold, answer)))+ except Exception:+ return None+++def _score_record(record: dict) -> None:+ boxed = last_boxed(record["solution_str"])+ predicted = as_integer(boxed)+ try:+ gold = int(str(record["ground_truth"]).strip())+ except ValueError:+ gold = None+ record["boxed"] = boxed+ record["predicted"] = predicted+ record["acc"] = float(predicted is not None and predicted == gold)+ record["no_answer"] = float(predicted is None)+ record["acc_math_verify"] = _math_verify(+ record["solution_str"], str(record["ground_truth"])+ )+++def grade(prepared, completions) -> List[dict]:+ records = []+ for item, completion in zip(prepared, completions, strict=True):+ example = item["example"]+ record = {+ "problem": example["problem"],+ "repeat": example["repeat"],+ "ground_truth": example["ground_truth"],+ "solution_str": str(completion.get("text") or ""),+ "gen_tokens": completion.get("gen_tokens"),+ "finish_reason": completion.get("finish_reason"),+ }+ _score_record(record)+ records.append(record)+ return records+++def score_jsonl(path: str) -> dict:+ """Re-score an ``aime25__<mode>.jsonl`` in place."""+ with open(path) as handle:+ records = [json.loads(line) for line in handle if line.strip()]+ for record in records:+ _score_record(record)+ with open(path, "w") as handle:+ for record in records:+ handle.write(json.dumps(record, ensure_ascii=False) + "\n")+ n = len(records)+ checked = [r for r in records if r.get("acc_math_verify") is not None]+ return {+ "n": n,+ "acc_mean": sum(r["acc"] for r in records) / n if n else 0.0,+ "n_no_answer": sum(int(r["no_answer"]) for r in records),+ "n_truncated": sum(1 for r in records if r.get("finish_reason") == "length"),+ "n_math_verify_disagree": sum(+ 1 for r in checked if r["acc_math_verify"] != r["acc"]+ ),+ "n_math_verify_checked": len(checked),+ }+++def main() -> None:+ import argparse++ parser = argparse.ArgumentParser(+ description="Re-score aime25__<mode>.jsonl on the boxed integer"+ )+ parser.add_argument("jsonl", nargs="+")+ args = parser.parse_args()+ for path in args.jsonl:+ summary = score_jsonl(path)+ print(+ f"{path}: n={summary['n']} acc={summary['acc_mean']:.4f} "+ f"no_answer={summary['n_no_answer']} "+ f"truncated={summary['n_truncated']} "+ f"mv_disagree={summary['n_math_verify_disagree']}"+ f"/{summary['n_math_verify_checked']}",+ file=sys.stdout,+ )+++if __name__ == "__main__":+ main()diff --git a/evaluation/compare.py b/evaluation/compare.pynew file mode 100644index 0000000..a06aeda--- /dev/null+++ b/evaluation/compare.py@@ -0,0 +1,437 @@+"""Merge sharded eval output and compare arms on the examples they share.++``run_eval`` scores one configuration at a time and writes one record per+example per shard. Deciding whether one configuration beats another is a+separate question, and the honest form of it is paired: every arm answers the+same benchmark, and examples differ from each other by far more than arms+differ from each other, so an unpaired interval is dominated by variation that+cancels.++Shards are strided (``examples[shard_index::num_shards]``), so an example's+position in the benchmark is ``shard_index + row * num_shards``. That is the+identity two arms are joined on; nothing else in the record identifies an+example, since ``id`` is null in the filtered MRCR file.++Usage:++ python -m evaluation.compare --root _local/accuracy \\+ --baseline bf16 --benchmark mrcr --mode bitdecoding-nvfp4+"""++from __future__ import annotations++import argparse+import glob+import json+import os+from typing import Any++import numpy as np+++METRIC_KEYS = {+ "mrcr": "ratio",+ "longbench_pro": "score",+ "aime": "acc",+ "aime25": "acc",+ "lcb": "pass1",+ "ruler_niah": "recall",+ "gpqa": "correct",+}+# Benchmarks whose rows are not independent: AIME is 30 problems sampled 32+# times each, so a row-level resample treats 960 draws as 960 problems and+# reports an interval about sqrt(32) too narrow. The unit of replication is+# the problem, and both the interval and the standard error use it.+CLUSTER_KEYS = {+ "aime": "problem", "aime25": "problem", "gpqa": "problem",+ "lcb": "problem_index",+}+BOOTSTRAP_RESAMPLES = 20_000+BOOTSTRAP_SEED = 42+++def _shard_file(shard_dir: str, benchmark: str, mode: str | None) -> str:+ """The one result file for this benchmark in this shard.++ A named ``mode`` is required to be there. Without one, the file is found by+ benchmark alone. Each arm directory must contain exactly one mode when+ ``--mode`` is omitted.+ """+ if mode is not None:+ raw = os.path.join(shard_dir, f"{benchmark}__{mode}.jsonl")+ scored = raw.removesuffix(".jsonl") + ".scored.jsonl"+ return scored if os.path.exists(scored) else raw+ found = sorted(glob.glob(os.path.join(shard_dir, f"{benchmark}__*.jsonl")))+ scored = [path for path in found if path.endswith(".scored.jsonl")]+ if scored:+ found = scored+ if not found:+ raise ValueError(f"{shard_dir} has no {benchmark}__*.jsonl")+ if len(found) > 1:+ raise ValueError(+ f"{shard_dir} holds {len(found)} modes for {benchmark} "+ f"({', '.join(os.path.basename(p) for p in found)}); pass --mode"+ )+ return found[0]+++def load_arm(+ arm_dir: str, benchmark: str, mode: str | None+) -> dict[Any, dict[str, Any]]:+ """One arm's per-example records, keyed by position in the benchmark."""+ records: dict[Any, dict[str, Any]] = {}+ shards = sorted(+ name+ for name in os.listdir(arm_dir)+ if name.startswith("shard_") and os.path.isdir(os.path.join(arm_dir, name))+ )+ if not shards:+ raise ValueError(f"{arm_dir} has no shard_* directories")+ for name in shards:+ shard_index = int(name.removeprefix("shard_"))+ path = _shard_file(os.path.join(arm_dir, name), benchmark, mode)+ if not os.path.exists(path):+ raise ValueError(f"arm {arm_dir!r} is missing {path}")+ with open(path) as handle:+ rows = [json.loads(line) for line in handle if line.strip()]+ for row_index, row in enumerate(rows):+ position = shard_index + row_index * len(shards)+ selection_identity = row.get("_selection_identity")+ identity = (+ json.dumps(selection_identity, sort_keys=True)+ if isinstance(selection_identity, dict)+ else (+ str(row["id"])+ if benchmark == "longbench_pro" and row.get("id") is not None+ else position+ )+ )+ if identity in records:+ raise ValueError(+ f"arm {arm_dir!r} repeats {benchmark} row identity {identity!r}"+ )+ records[identity] = row+ return records+++def _metric_value(row: dict[str, Any], benchmark: str) -> float:+ key = METRIC_KEYS[benchmark]+ value = row.get(key)+ # Accept the older LongBench-Pro metric name when reading saved results.+ if value is None and benchmark == "longbench_pro":+ value = row.get("f1")+ return float(value or 0.0)+++def _cluster_means(delta: np.ndarray, clusters: np.ndarray | None) -> np.ndarray:+ """One number per unit of replication: a row, or a cluster of them."""+ if clusters is None:+ return delta+ labels = np.unique(clusters)+ return np.asarray([delta[clusters == label].mean() for label in labels])+++def paired_bootstrap(+ delta: np.ndarray,+ *,+ clusters: np.ndarray | None = None,+ resamples: int = BOOTSTRAP_RESAMPLES,+ seed: int = BOOTSTRAP_SEED,+) -> tuple[float, float]:+ """Percentile interval for the mean of example-wise differences.++ With ``clusters``, whole clusters are resampled rather than rows. The+ design here is balanced, so the mean over rows equals the mean over+ cluster means and only the interval changes.+ """+ units = _cluster_means(delta, clusters)+ generator = np.random.default_rng(seed)+ means = []+ remaining = resamples+ while remaining:+ take = min(256, remaining)+ indices = generator.integers(0, units.size, size=(take, units.size))+ means.append(units[indices].mean(axis=1))+ remaining -= take+ means = np.concatenate(means)+ return float(np.quantile(means, 0.025)), float(np.quantile(means, 0.975))+++def _linear_slope(x: np.ndarray, y: np.ndarray) -> float:+ centered = x - x.mean()+ denominator = float(np.square(centered).sum())+ if denominator == 0.0:+ raise ValueError("length-slope input needs at least two prompt lengths")+ return float(np.dot(centered, y - y.mean()) / denominator)+++def paired_slope_bootstrap(+ prompt_tokens: np.ndarray,+ delta: np.ndarray,+ *,+ resamples: int = BOOTSTRAP_RESAMPLES,+ seed: int = BOOTSTRAP_SEED,+ batch_size: int = 256,+) -> tuple[float, float, float]:+ """Slope and percentile CI for paired delta versus log2 prompt length.++ Rows are resampled as paired ``(length, delta)`` observations. Resamples+ are generated in bounded batches so 20,000 resamples of the full long-+ context pool do not allocate a multi-gigabyte index matrix.+ """+ tokens = np.asarray(prompt_tokens, dtype=float)+ values = np.asarray(delta, dtype=float)+ if tokens.shape != values.shape or tokens.ndim != 1:+ raise ValueError("prompt_tokens and delta must be same-length vectors")+ if tokens.size < 2 or np.any(tokens <= 0):+ raise ValueError("length-slope input needs at least two positive lengths")+ x = np.log2(tokens)+ observed = _linear_slope(x, values)+ generator = np.random.default_rng(seed)+ slopes: list[np.ndarray] = []+ remaining = resamples+ while remaining:+ take = min(batch_size, remaining)+ indices = generator.integers(0, x.size, size=(take, x.size))+ bx = x[indices]+ by = values[indices]+ bx = bx - bx.mean(axis=1, keepdims=True)+ by = by - by.mean(axis=1, keepdims=True)+ denominator = np.square(bx).sum(axis=1)+ valid = denominator > 0+ slopes.append((bx[valid] * by[valid]).sum(axis=1) / denominator[valid])+ remaining -= take+ sampled = np.concatenate(slopes)+ if sampled.size < max(100, resamples // 2):+ raise ValueError("too many degenerate length-slope bootstrap resamples")+ low, high = np.quantile(sampled, [0.025, 0.975])+ return observed, float(low), float(high)+++def compare_length_slope(+ root: str,+ *,+ baseline: str,+ arm: str,+ mode: str | None = None,+ benchmarks: tuple[str, ...] = ("longbench_pro", "mrcr"),+ resamples: int = BOOTSTRAP_RESAMPLES,+ seed: int = BOOTSTRAP_SEED,+ output_path: str | None = None,+) -> dict[str, Any]:+ """Registered long-context evaluation length-slope test pooled over long benchmarks."""+ tokens: list[float] = []+ deltas: list[float] = []+ counts: dict[str, int] = {}+ for benchmark in benchmarks:+ base_rows = load_arm(os.path.join(root, baseline), benchmark, mode)+ arm_rows = load_arm(os.path.join(root, arm), benchmark, mode)+ shared = sorted(set(base_rows) & set(arm_rows))+ counts[benchmark] = len(shared)+ for index in shared:+ prompt_tokens = base_rows[index].get("prompt_tokens")+ if prompt_tokens is None:+ raise ValueError(+ f"{benchmark} row {index} has no prompt_tokens for slope"+ )+ tokens.append(float(prompt_tokens))+ deltas.append(+ _metric_value(arm_rows[index], benchmark)+ - _metric_value(base_rows[index], benchmark)+ )+ observed, low, high = paired_slope_bootstrap(+ np.asarray(tokens), np.asarray(deltas),+ resamples=resamples, seed=seed,+ )+ summary: dict[str, Any] = {+ "baseline": baseline,+ "arm": arm,+ "benchmarks": list(benchmarks),+ "n_examples": len(tokens),+ "n_by_benchmark": counts,+ "x": "log2(prompt_tokens)",+ "slope_per_doubling": observed,+ f"bootstrap_95_ci_seed{seed}_{resamples}": [low, high],+ }+ if output_path:+ os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)+ with open(output_path, "w") as handle:+ json.dump(summary, handle, indent=2)+ return summary+++def compare(+ root: str,+ benchmark: str,+ mode: str,+ *,+ baseline: str,+ arms: list[str] | None = None,+ output_path: str | None = None,+) -> dict[str, Any]:+ metric = METRIC_KEYS[benchmark]+ names = arms or sorted(+ name+ for name in os.listdir(root)+ if os.path.isdir(os.path.join(root, name))+ )+ # Naming a subset of arms is how you compare two of many, and the baseline+ # is what they are compared against rather than one of them, so asking for+ # it twice would be the odd spelling.+ if baseline not in names:+ names = [baseline, *names]+ loaded = {+ name: load_arm(os.path.join(root, name), benchmark, mode)+ for name in names+ }+ shared = sorted(set.intersection(*(set(arm) for arm in loaded.values())))+ if not shared:+ raise ValueError("the arms share no examples")++ scores = {+ name: np.asarray(+ [_metric_value(arm[index], benchmark) for index in shared]+ )+ for name, arm in loaded.items()+ }+ cluster_key = CLUSTER_KEYS.get(benchmark)+ clusters = (+ np.asarray([loaded[baseline][index][cluster_key] for index in shared])+ if cluster_key+ else None+ )+ summary: dict[str, Any] = {+ "benchmark": benchmark,+ "mode": mode,+ "baseline": baseline,+ "n_examples": len(shared),+ "n_clusters": (int(np.unique(clusters).size) if clusters is not None+ else None),+ "cluster_key": cluster_key,+ "arms": {},+ "paired_vs_baseline": {},+ }+ interval_key = f"bootstrap_95_ci_seed{BOOTSTRAP_SEED}_{BOOTSTRAP_RESAMPLES}"+ for name, value in scores.items():+ low, high = paired_bootstrap(value, clusters=clusters)+ summary["arms"][name] = {+ "mean": float(value.mean()),+ "n": int(value.size),+ "complete": len(loaded[name]) == len(shared),+ interval_key: [low, high],+ }+ for name, value in scores.items():+ if name == baseline:+ continue+ delta = value - scores[baseline]+ low, high = paired_bootstrap(delta, clusters=clusters)+ units = _cluster_means(delta, clusters)+ se = float(units.std(ddof=1) / np.sqrt(units.size))+ summary["paired_vs_baseline"][name] = {+ "mean_delta": float(delta.mean()),+ "se_delta": se,+ "normal_95_ci": [+ float(delta.mean() - 1.96 * se),+ float(delta.mean() + 1.96 * se),+ ],+ f"bootstrap_95_ci_seed{BOOTSTRAP_SEED}_{BOOTSTRAP_RESAMPLES}": [+ low,+ high,+ ],+ "wins": int((delta > 0).sum()),+ "losses": int((delta < 0).sum()),+ "ties": int((delta == 0).sum()),+ }++ if benchmark == "longbench_pro":+ summary["by_token_length"] = {}+ buckets = sorted({+ str(loaded[baseline][identity].get("token_length"))+ for identity in shared+ }, key=lambda value: int(value[:-1]) if value.endswith("k") else value)+ for bucket in buckets:+ positions = np.asarray([+ index for index, identity in enumerate(shared)+ if str(loaded[baseline][identity].get("token_length")) == bucket+ ])+ bucket_scores = {+ name: values[positions] for name, values in scores.items()+ }+ bucket_summary = {+ "n": int(positions.size),+ "arms": {+ name: {"mean": float(values.mean()), "n": int(values.size)}+ for name, values in bucket_scores.items()+ },+ "paired_vs_baseline": {},+ }+ for name, values in bucket_scores.items():+ if name == baseline:+ continue+ delta = values - bucket_scores[baseline]+ low, high = paired_bootstrap(delta)+ bucket_summary["paired_vs_baseline"][name] = {+ "mean_delta": float(delta.mean()),+ f"bootstrap_95_ci_seed{BOOTSTRAP_SEED}_{BOOTSTRAP_RESAMPLES}": [+ low,+ high,+ ],+ "wins": int((delta > 0).sum()),+ "losses": int((delta < 0).sum()),+ "ties": int((delta == 0).sum()),+ }+ summary["by_token_length"][bucket] = bucket_summary++ if output_path:+ os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)+ with open(output_path, "w") as handle:+ json.dump(summary, handle, indent=2)+ return summary+++def _print(summary: dict[str, Any]) -> None:+ clustered = (+ f" over {summary['n_clusters']} {summary['cluster_key']}s"+ if summary.get("n_clusters")+ else ""+ )+ print(f"{summary['benchmark']} n={summary['n_examples']}{clustered}")+ for name, arm in summary["arms"].items():+ print(f" {name:12s} {arm['mean']:.4f}")+ key = f"bootstrap_95_ci_seed{BOOTSTRAP_SEED}_{BOOTSTRAP_RESAMPLES}"+ for name, delta in summary["paired_vs_baseline"].items():+ low, high = delta[key]+ print(+ f" {name:12s} vs {summary['baseline']}: "+ f"{delta['mean_delta']:+.5f} "+ f"boot95=[{low:+.5f}, {high:+.5f}] "+ f"W/L/T={delta['wins']}/{delta['losses']}/{delta['ties']}"+ )+++def main() -> None:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--root", required=True)+ parser.add_argument("--benchmark", default="mrcr", choices=sorted(METRIC_KEYS))+ parser.add_argument("--mode", default=None,+ help="the KV mode whose file to read in each arm "+ "directory; omit when the arm is the mode and "+ "each directory holds exactly one")+ parser.add_argument("--baseline", required=True)+ parser.add_argument("--arms", nargs="*", default=None)+ parser.add_argument("--output-path", default=None)+ args = parser.parse_args()+ _print(+ compare(+ args.root,+ args.benchmark,+ args.mode,+ baseline=args.baseline,+ arms=args.arms,+ output_path=args.output_path,+ )+ )+++if __name__ == "__main__":+ main()diff --git a/evaluation/config.py b/evaluation/config.pynew file mode 100644index 0000000..765b9fc--- /dev/null+++ b/evaluation/config.py@@ -0,0 +1,365 @@+"""Static configuration for the BitDecoding-FP4 evaluation.++One in-process vLLM engine is stood up per KV-cache mode and every requested+benchmark is scored against it. The modes include:++ * ``bf16`` -> BF16 KV baseline (``kv_cache_dtype="auto"``)+ * ``fp8`` -> vLLM FP8 e4m3 KV cache with unit scales+ * ``turboquant-4bit-nc`` -> vLLM TurboQuant 4-bit keys and values+ * ``turboquant-4bit-nc-all-layers`` -> the same preset on every layer+ * ``nvfp4`` -> official vLLM NVFP4 KV cache+ * ``bitdecoding-nvfp4`` -> this repository's BitDecoding-NVFP4 recipe+ * ``bitdecoding-mxfp4`` -> MXFP4 scales with the same serving recipe++``bitdecoding-nvfp4`` is the method: CUSTOM backend, 16-token BF16 sink,+K centering, ``{4,6}`` block scales for K with the default ``amax6`` for V+and Q, two-term query, and Hadamard rotation. It+calibrates static key smoothing and the probability threshold together before+scoring, or reuses the matching calibration saved with the evaluation output.++``EVALUATION`` defines the default 128K run. Dataset paths are relative to+the repository's ``datasets/`` directory, populated by ``tools/prepare_data.py``.+"""+from __future__ import annotations++import copy+import json+import math+import os++EVALUATION = {+ "max_model_len": 131_072,+ "max_num_seqs": 1,+ "longbench_prompt_budget": 98_048,+ "longbench_excluded_buckets": ["256k"],+ "mrcr_cache": "mrcr/_filtered_le98048.jsonl",+}++RECIPE = {+ "dual_residual_query_quantization": True,+ "underflow_aware_probability_scaling": {+ "enabled": True,+ "c": 8,+ "tau": "calibrate",+ },+ "outlier_aware_key_quantization": {+ "key_smoothing": True,+ "hadamard_rotation": True,+ "adaptive_block_scale_search": True,+ },+}+++def recipe_additional_config(recipe: dict, *, key_smoothing: str | None = None) -> dict:+ keys = recipe["outlier_aware_key_quantization"]+ return {+ "nvfp4_key_smoothing": key_smoothing if keys["key_smoothing"] else None,+ "nvfp4_scale_mode": {+ "key": "four_six" if keys["adaptive_block_scale_search"] else "amax6",+ "value": "amax6", "query": "amax6",+ },+ "nvfp4_query_terms": 2 if recipe["dual_residual_query_quantization"] else 1,+ "nvfp4_rotation": "hadamard" if keys["hadamard_rotation"] else "none",+ }+++def needs_tau_calibration(recipe: dict) -> bool:+ probability = recipe["underflow_aware_probability_scaling"]+ return probability["enabled"] and probability["tau"] == "calibrate"+++def probability_environment(recipe: dict, *, calibration: bool = False) -> dict:+ probability = recipe["underflow_aware_probability_scaling"]+ if not probability["enabled"]:+ return {"NVFP4_P_SCALE_MODE": "amax6", "NVFP4_P_SCALE_TAU": None}+ if probability["c"] not in (1, 8):+ raise ValueError("the probability kernel supports c=1 or c=8")+ tau = probability["tau"]+ if tau == "calibrate":+ if not calibration:+ raise ValueError("tau must be calibrated before evaluation or throughput measurement")+ tau = None+ elif type(tau) not in (int, float) or not math.isfinite(tau) or not 0 <= tau <= 448:+ raise ValueError("tau must be 'calibrate' or a finite number between 0 and 448")+ return {+ "NVFP4_P_SCALE_MODE": "amax2_or_amax6_c8" if probability["c"] == 8 else "amax2_or_amax6",+ "NVFP4_P_SCALE_TAU": None if tau is None else repr(float(tau)),+ }+++# Repo root == parent of this ``evaluation/`` package.+REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))++# Use a Hub model ID or a local checkpoint path.+MODEL_PATH = os.environ.get("MODEL_PATH", "Qwen/Qwen3-4B-Thinking-2507")+# Evaluation datasets (staged by tools/prepare_data.py). Override with BITDECODING_DATA_DIR.+DATA_DIR = os.environ.get("BITDECODING_DATA_DIR", os.path.join(REPO_ROOT, "datasets"))+# Launchers can override these defaults for individual evaluation shards.+MAX_MODEL_LEN = int(os.environ.get("BITDECODING_MAX_MODEL_LEN", EVALUATION["max_model_len"]))+LONGBENCH_PROMPT_BUDGET = int(+ os.environ.get("BITDECODING_LONGBENCH_PROMPT_BUDGET", EVALUATION["longbench_prompt_budget"])+)++# Use the same scheduler and CUDA graph settings across cache modes.+PARITY_ENGINE_KWARGS = {+ "max_num_seqs": int(os.environ.get(+ "BITDECODING_LONG_CONTEXT_MAX_NUM_SEQS", EVALUATION["max_num_seqs"]+ )),+ "block_size": 128,+ "enable_prefix_caching": False,+ "enable_chunked_prefill": False,+ "max_num_batched_tokens": max(40_960, MAX_MODEL_LEN),+ "pipeline_parallel_size": 1,+ "enforce_eager": False,+}+LONG_CONTEXT_ENGINE_KWARGS = dict(PARITY_ENGINE_KWARGS)++# Shared calibration files; the accuracy runner creates missing key means.+SIDECAR_DIR = os.environ.get(+ "BITDECODING_SIDECAR_DIR", os.path.join(REPO_ROOT, "_local", "sidecars")+)+KEY_SHIFT_SIDECAR = os.environ.get(+ "BITDECODING_KEY_SHIFT_SIDECAR",+ os.path.join(SIDECAR_DIR, "key_shift.safetensors"),+)++# Windows above this threshold use the long-context sampling settings.+SHORT_CONTEXT_MAX_MODEL_LEN = 40_960++KV_MODES = {+ "bf16": {+ "kv_cache_dtype": "auto",+ "label": "BF16 KV (baseline)",+ },+ "fp8": {+ "kv_cache_dtype": "fp8",+ "label": "vLLM FP8 e4m3 KV (unit scales)",+ },+ "turboquant-4bit-nc": {+ "kv_cache_dtype": "turboquant_4bit_nc",+ "label": "vLLM TurboQuant 4-bit NC",+ },+ "turboquant-4bit-nc-all-layers": {+ "kv_cache_dtype": "turboquant_4bit_nc",+ "turboquant_all_layers": True,+ "label": "vLLM TurboQuant 4-bit NC (all layers)",+ },+ "nvfp4": {+ "kv_cache_dtype": "nvfp4",+ "label": "NVFP4 KV cache (official vLLM)",+ },+ "bitdecoding-nvfp4": {+ "kv_cache_dtype": "nvfp4",+ "engine_kwargs": PARITY_ENGINE_KWARGS,+ "custom_backend": True,+ "sink_tokens": 16,+ "scale_mode": recipe_additional_config(RECIPE)["nvfp4_scale_mode"],+ "key_scale_geometry": os.environ.get(+ "BITDECODING_KEY_SCALE_GEOMETRY", "local"+ ),+ "query_terms": recipe_additional_config(RECIPE)["nvfp4_query_terms"],+ "rotation": os.environ.get("BITDECODING_CUSTOM_ROTATION", recipe_additional_config(RECIPE)["nvfp4_rotation"]),+ "key_shift": (KEY_SHIFT_SIDECAR+ if RECIPE["outlier_aware_key_quantization"]["key_smoothing"] else None),+ "label": "BitDecoding-NVFP4",+ },+ "bitdecoding-mxfp4": {+ "kv_cache_dtype": "nvfp4",+ "engine_kwargs": PARITY_ENGINE_KWARGS,+ "custom_backend": True,+ "sink_tokens": 16,+ "scale_mode": "mxfp4_search",+ "key_scale_geometry": "local",+ "query_terms": recipe_additional_config(RECIPE)["nvfp4_query_terms"],+ "rotation": "hadamard",+ "key_shift": (KEY_SHIFT_SIDECAR+ if RECIPE["outlier_aware_key_quantization"]["key_smoothing"] else None),+ "label": "BitDecoding-MXFP4",+ },+}+++def model_family(model_path=None):+ """Read the model family from its config, falling back to the checkpoint name."""+ try:+ with open(os.path.join(model_path or MODEL_PATH, "config.json")) as handle:+ value = json.load(handle)+ if "glm" in json.dumps(value).lower():+ return "glm"+ except (OSError, TypeError, json.JSONDecodeError):+ pass+ return "glm" if "glm" in str(model_path or "").lower() else "qwen"+++def chat_template_kwargs(model_path, thinking):+ return {"enable_thinking": thinking}+++def resolve_mode(name: str, model_path=None) -> dict:+ """A copy of ``KV_MODES[name]`` with long-context engine overlays applied."""+ mode = copy.deepcopy(KV_MODES[name])+ seqs = int(os.environ.get(+ "BITDECODING_LONG_CONTEXT_MAX_NUM_SEQS", EVALUATION["max_num_seqs"]+ ))+ kwargs = dict(LONG_CONTEXT_ENGINE_KWARGS)+ kwargs.update(mode.get("engine_kwargs") or {})+ # Disable FP8 autotuning to avoid FA2 paths without FP8 prefill support.+ if name == "fp8":+ kwargs["attention_backend"] = "FLASHINFER"+ kwargs["enable_flashinfer_autotune"] = False+ elif name not in {+ "turboquant-4bit-nc",+ "turboquant-4bit-nc-all-layers",+ "bitdecoding-nvfp4",+ "bitdecoding-mxfp4",+ }:+ kwargs["attention_backend"] = "FLASHINFER"+ kwargs["max_num_seqs"] = seqs+ kwargs["max_num_batched_tokens"] = max(+ int(kwargs.get("max_num_batched_tokens") or 0), MAX_MODEL_LEN+ )+ mode["engine_kwargs"] = kwargs+ return mode++# Qwen thinking-mode sampling; use identical per-row seeds across cache modes.+THINKING_SAMPLING = {"temperature": 0.6, "top_p": 0.95, "top_k": 20, "min_p": 0.0}+MODEL_THINKING_SAMPLING = {+ "qwen": THINKING_SAMPLING,+ "glm": {"temperature": 1.0, "top_p": 0.95, "min_p": 0.0},+}++# ``max_new_tokens`` may be this word instead of a number, meaning "whatever is+# left of the window once this row's prompt is in it". One constant cannot say+# that for a benchmark whose prompts run from 8k to 28.6k: set for the longest+# prompt it truncates the median row, set for the median row it overruns the+# window on the longest. Every arm sees the same prompts, so every arm gets the+# same per-row budget and the arms stay paired.+FILL_WINDOW = "fill_window"+# Slack for the chat template's own tokens, which ``prompt_tokens`` counts+# before the template is applied and so does not include.+GENERATION_MARGIN = 256++BENCH_DEFAULTS = {+ "longbench_pro": {"temperature": 0.0, "top_p": 1.0, "max_new_tokens": 4096},+ "mrcr": {"temperature": 0.0, "top_p": 1.0, "max_new_tokens": 4096},+ "aime": {**THINKING_SAMPLING, "max_new_tokens": 38_912},+ "aime25": {**THINKING_SAMPLING, "max_new_tokens": 38_912},+ "lcb": {**THINKING_SAMPLING, "max_new_tokens": 39_168},+ "ruler_niah": {**THINKING_SAMPLING, "max_new_tokens": 8_192},+ "gpqa": {**THINKING_SAMPLING, "max_new_tokens": 81_920},+}++# Thinking-mode sampling for the greedy benchmarks. Selected by --thinking+# rather than by redefining the greedy defaults above.+THINKING_BENCH_OVERRIDES = {+ "longbench_pro": {**THINKING_SAMPLING, "max_new_tokens": FILL_WINDOW},+ "mrcr": {**THINKING_SAMPLING, "max_new_tokens": FILL_WINDOW},+ "ruler_niah": {**THINKING_SAMPLING, "max_new_tokens": 8_192},+ "gpqa": {**THINKING_SAMPLING, "max_new_tokens": 81_920},+}++def configure_data_dir(data_dir: str) -> None:+ """Resolve dataset paths and their staging checks from one root."""+ global DATA_DIR, LONGBENCH_PRO_DIR, MRCR_FILTERED_CACHE+ global AIME24_PARQUET, AIME25_PARQUET, LCB_PROMPTS, RULER_NIAH_PROMPTS+ global GPQA_PATH, BENCH_REQUIREMENTS++ DATA_DIR = os.path.abspath(os.path.expanduser(data_dir))+ LONGBENCH_PRO_DIR = os.path.join(DATA_DIR, "longbench_pro")+ MRCR_FILTERED_CACHE = os.environ.get(+ "BITDECODING_MRCR_FILTERED_CACHE",+ os.path.join(DATA_DIR, EVALUATION["mrcr_cache"]),+ )+ AIME24_PARQUET = os.path.join(DATA_DIR, "aime24", "aime24.parquet")+ AIME25_PARQUET = os.path.join(DATA_DIR, "aime25", "aime25.parquet")+ LCB_PROMPTS = os.path.join(DATA_DIR, "lcb_v6", "prompts.jsonl")+ RULER_NIAH_PROMPTS = os.path.join(DATA_DIR, "ruler", "niah_122880.jsonl")+ GPQA_PATH = os.environ.get(+ "BITDECODING_GPQA_PATH", os.path.join(DATA_DIR, "gpqa", "gpqa_diamond.parquet")+ )+ BENCH_REQUIREMENTS = {+ "longbench_pro": [("LongBench-Pro dataset", LONGBENCH_PRO_DIR, "dir")],+ "mrcr": [("MRCR filtered cache", MRCR_FILTERED_CACHE, "file")],+ "aime": [("AIME 2024 parquet", AIME24_PARQUET, "file")],+ "aime25": [("AIME 2025 parquet", AIME25_PARQUET, "file")],+ "lcb": [("LiveCodeBench v6 prompts", LCB_PROMPTS, "file")],+ "ruler_niah": [("RULER NIAH prompts", RULER_NIAH_PROMPTS, "file")],+ "gpqa": [("GPQA Diamond data", GPQA_PATH, "file")],+ }+++configure_data_dir(DATA_DIR)+BENCH_DEPENDENCIES = {}++LONG_CONTEXT_BENCH_OVERRIDES = {+ "aime": {**THINKING_SAMPLING, "max_new_tokens": 81_920},+ "aime25": {**THINKING_SAMPLING, "max_new_tokens": 81_920},+ "lcb": {**THINKING_SAMPLING, "max_new_tokens": 81_920},+ "longbench_pro": {**THINKING_SAMPLING, "max_new_tokens": FILL_WINDOW},+ "mrcr": {**THINKING_SAMPLING, "max_new_tokens": FILL_WINDOW},+ "ruler_niah": {**THINKING_SAMPLING, "max_new_tokens": 8_192},+ "gpqa": {**THINKING_SAMPLING, "max_new_tokens": 81_920},+}+++def _model_available() -> bool:+ """A local snapshot must hold a config.json; a bare Hub id vLLM downloads.++ A directory that is actually there is local whatever it is spelled like:+ ``_local/models/Qwen3-8B`` is neither absolute nor ``./``-prefixed, and+ reading it as a Hub id rejected a checkpoint that was sitting right there.+ A path that does not exist still falls through to the Hub-id shape, so a+ mistyped local path is reported as missing rather than downloaded.+ """+ if (os.path.isdir(MODEL_PATH) or os.path.isabs(MODEL_PATH)+ or MODEL_PATH.startswith((".", "~"))):+ return os.path.isfile(os.path.join(MODEL_PATH, "config.json"))+ return MODEL_PATH.count("/") == 1+++def verify_prestaging(benchmarks=None) -> list:+ """Return a list of human-readable problems; empty list == ready to run.++ ``benchmarks`` restricts the check to what will actually be read. Passing+ nothing keeps the old behaviour of demanding everything.+ """+ problems = []+ if not _model_available():+ problems.append(f"model checkpoint missing at {MODEL_PATH} (set MODEL_PATH)")+ names = list(BENCH_REQUIREMENTS) if benchmarks is None else list(benchmarks)+ for name in names:+ for description, path, kind in BENCH_REQUIREMENTS.get(name, []):+ present = os.path.isdir(path) if kind == "dir" else os.path.isfile(path)+ if not present:+ problems.append(f"{description} missing at {path}")+ return problems+++def verify_dependencies(benchmarks) -> list:+ """Problems with the packages the requested benchmarks score with."""+ import importlib.util++ problems = []+ for name in benchmarks:+ for module, purpose in BENCH_DEPENDENCIES.get(name, []):+ if importlib.util.find_spec(module) is None:+ problems.append(+ f"benchmark {name!r} scores with {module!r} ({purpose}), "+ "which is not installed"+ )+ return problems+++def verify_sidecars(modes) -> list:+ """Problems with the calibration sidecars the requested modes name."""+ problems = []+ for mode in modes:+ entry = KV_MODES.get(mode, {})+ for key in ("key_shift",):+ path = entry.get(key)+ if isinstance(path, str) and path != "auto" and not os.path.isfile(path):+ problems.append(+ f"mode {mode!r} wants the {key} sidecar at {path}, which is "+ "not there (run the accuracy evaluation to calibrate it)"+ )+ return problemsdiff --git a/evaluation/gpqa.py b/evaluation/gpqa.pynew file mode 100644index 0000000..020c195--- /dev/null+++ b/evaluation/gpqa.py@@ -0,0 +1,115 @@+"""GPQA Diamond multiple choice at Avg@8."""+from __future__ import annotations++import json+import random+import re+from typing import List++from . import config++METRIC_KEY = "correct"+REPEATS = 8+SHUFFLE_SEED = 0+_ANSWER_RE = re.compile(r"Answer[ \t]*:[ \t]*\$?([A-D])\$?", re.IGNORECASE)+++def _rows(path):+ if path.endswith(".jsonl"):+ with open(path) as handle:+ return [json.loads(line) for line in handle if line.strip()]+ import pandas as pd+ frame = pd.read_csv(path) if path.endswith(".csv") else pd.read_parquet(path)+ return frame.to_dict(orient="records")+++def _normalise(row):+ question = row.get("Question", row.get("question"))+ choices = row.get("Choices", row.get("choices"))+ answer = row.get("Answer", row.get("answer"))+ if choices is None and row.get("Correct Answer") is not None:+ choices = [+ row["Correct Answer"],+ row.get("Incorrect Answer 1"),+ row.get("Incorrect Answer 2"),+ row.get("Incorrect Answer 3"),+ ]+ answer = 0+ if question is None or choices is None or answer is None:+ raise ValueError("GPQA row must contain question, choices, and answer")+ if isinstance(choices, str):+ choices = json.loads(choices)+ choices = list(choices)+ if len(choices) != 4:+ raise ValueError(f"GPQA expects four choices, got {len(choices)}")+ if isinstance(answer, str) and answer.strip().upper() in "ABCD":+ answer_index = "ABCD".index(answer.strip().upper())+ else:+ answer_index = int(answer)+ return str(question), choices, answer_index+++def load_examples(path: str = None) -> List[dict]:+ examples = []+ rng = random.Random(SHUFFLE_SEED)+ for problem, row in enumerate(_rows(path or config.GPQA_PATH)):+ question, choices, answer_index = _normalise(row)+ for repeat in range(REPEATS):+ order = rng.sample(range(4), 4)+ content = (+ "Answer the following multiple choice question. The last line "+ "of your response should be of the following format: 'Answer: "+ "$LETTER' (without quotes) where LETTER is one of ABCD. Think "+ "step by step before answering.\n\n"+ f"{question}\n\n" ++ "\n".join(+ f"{letter}) {choices[index]}"+ for letter, index in zip("ABCD", order)+ )+ )+ examples.append({+ "problem": problem,+ "repeat": repeat,+ "messages": [{"role": "user", "content": content}],+ "correct_letter": "ABCD"[order.index(answer_index)],+ "choice_order": order,+ })+ return examples+++def build(examples):+ return [ex["messages"] for ex in examples], [{"example": ex} for ex in examples]+++def seeds(prepared, base_seed: int):+ return [+ base_seed + 1_000_003 * item["example"]["problem"] + item["example"]["repeat"]+ for item in prepared+ ]+++def extract_letter(text: str):+ text = str(text or "")+ if "</think>" not in text:+ return None+ matches = _ANSWER_RE.findall(text.split("</think>", 1)[1])+ return matches[-1].upper() if matches else None+++def grade(prepared, completions):+ records = []+ for item, completion in zip(prepared, completions, strict=True):+ example = item["example"]+ predicted = extract_letter(completion.get("text") or "")+ records.append({+ "problem": example["problem"],+ "repeat": example["repeat"],+ "correct_letter": example["correct_letter"],+ "predicted_letter": predicted,+ "correct": int(predicted == example["correct_letter"]),+ "answered": int(predicted is not None),+ "raw_output": str(completion.get("text") or ""),+ "gen_tokens": completion.get("gen_tokens"),+ "finish_reason": completion.get("finish_reason"),+ })+ return recordsdiff --git a/evaluation/lcb.py b/evaluation/lcb.pynew file mode 100644index 0000000..fe7fcad--- /dev/null+++ b/evaluation/lcb.py@@ -0,0 +1,143 @@+"""LiveCodeBench v6 code generation (Avg@4 pass@1).++Three things about the official harness shape this module.++The official loader is a dataset script, which ``datasets`` 4.0 removed, so+this module never imports ``lcb_runner``. The benchmark is read once into the+flat JSONL this module consumes -- prompts only, no test cases. LiveCodeBench+stays on ``PYTHONPATH`` for ``tools/score_lcb.py``; it is not pip-installed.++It does no extraction. ``custom_evaluator`` passes the string it is given as+both the raw output and the extracted code, so what we submit must already be+runnable Python. :func:`extract_code` is LCB's own rule, reproduced exactly:+the content between the last two lines containing a fence, and the empty string+when there are fewer than two.++It zips our list against the benchmark in load order, sorting only when the+dict form is used. Rows therefore carry ``bench_index``, their position in that+load order, and ``tools/score_lcb.py`` submits the positional form. Nothing then+depends on ``question_id`` sorting the way the benchmark happens to be ordered.++Scoring is deferred for a fourth reason: it executes model-written code, which+must not happen in a process holding a CUDA context. Generation writes+``pass1=None``; ``tools/score_lcb.py`` fills it in.+Each of the 175 problems is generated four times with stable per-row seeds;+the sandbox grader evaluates every repeat and averages by problem.+"""+from __future__ import annotations++import json+from typing import List++from . import config+from .scoring import strip_thinking+++METRIC_KEY = "pass1"+REPEATS = 4++# Reproduced from lcb_runner/prompts/code_generation.py::PromptConstants so the+# prompt is byte-identical to the one the published numbers were produced with.+SYSTEM_MESSAGE = (+ "You are an expert Python programmer. You will be given a question "+ "(problem specification) and will generate a correct Python program that "+ "matches the specification and passes all tests."+)+FORMATTING_WITH_STARTER_CODE = (+ "You will use the following starter code to write the solution to the "+ "problem and enclose your code within delimiters."+)+FORMATTING_WITHOUT_STARTER_CODE = (+ "Read the inputs from stdin solve the problem and write the answer to "+ "stdout (do not directly test on the sample inputs). Enclose your code "+ "within delimiters as follows. Ensure that when the python program runs, "+ "it reads the inputs, runs the algorithm and writes output to STDOUT."+)++def load_examples(path: str = None) -> List[dict]:+ rows = []+ with open(path or config.LCB_PROMPTS) as handle:+ for line in handle:+ line = line.strip()+ if line:+ rows.append(json.loads(line))+ # Load order is the contract with the evaluator, so it is restored here+ # rather than trusted to have survived the file.+ rows.sort(key=lambda row: row["bench_index"])+ repeated = []+ for row in rows:+ problem_index = row["bench_index"]+ for repeat in range(REPEATS):+ item = dict(row)+ item["problem_index"] = problem_index+ item["repeat"] = repeat+ item["bench_index"] = problem_index * REPEATS + repeat+ repeated.append(item)+ return repeated+++def seeds(prepared, base_seed: int):+ return [+ base_seed + 1_000_003 * item["example"]["problem_index"] + item["example"]["repeat"]+ for item in prepared+ ]+++def build_prompt(example: dict) -> str:+ prompt = f"### Question:\n{example['question_content']}\n\n"+ if example.get("starter_code"):+ prompt += f"### Format: {FORMATTING_WITH_STARTER_CODE}\n"+ prompt += f"```python\n{example['starter_code']}\n```\n\n"+ else:+ prompt += f"### Format: {FORMATTING_WITHOUT_STARTER_CODE}\n"+ prompt += "```python\n# YOUR CODE HERE\n```\n\n"+ prompt += "### Answer: (use the provided format with backticks)\n\n"+ return prompt+++def build(examples):+ conversations, prepared = [], []+ for example in examples:+ conversations.append([+ {"role": "system", "content": SYSTEM_MESSAGE},+ {"role": "user", "content": build_prompt(example)},+ ])+ prepared.append({"example": example})+ return conversations, prepared+++def extract_code(model_output: str) -> str:+ """LCB's own extraction: between the last two fence lines, else empty.++ The thinking block is stripped first. A reasoning model fences code inside+ its reasoning, and the last two fences in the whole string would otherwise+ be able to land there rather than on the answer.+ """+ answer = strip_thinking(model_output)+ lines = answer.split("\n")+ fences = [i for i, line in enumerate(lines) if "```" in line]+ if len(fences) < 2:+ return ""+ return "\n".join(lines[fences[-2] + 1:fences[-1]])+++def grade(prepared, completions) -> List[dict]:+ records = []+ for item, completion in zip(prepared, completions, strict=True):+ example = item["example"]+ text = str(completion.get("text") or "")+ records.append({+ "bench_index": example["bench_index"],+ "problem_index": example["problem_index"],+ "repeat": example["repeat"],+ "question_id": example["question_id"],+ "platform": example.get("platform"),+ "difficulty": example.get("difficulty"),+ "contest_date": example.get("contest_date"),+ "code": extract_code(text),+ "raw_output": text,+ "pass1": None,+ "gen_tokens": completion.get("gen_tokens"),+ "finish_reason": completion.get("finish_reason"),+ })+ return recordsdiff --git a/evaluation/longbench_pro.py b/evaluation/longbench_pro.pynew file mode 100644index 0000000..b0c2a06--- /dev/null+++ b/evaluation/longbench_pro.py@@ -0,0 +1,357 @@+"""LongBench-Pro benchmark with its task-specific metrics.++Loads all secondary tasks, sends each prompt as a single-user-message chat,+and routes scoring on ``secondary_task``. T4 generations are deliberately+left unscored until the generation engines have exited.+"""+from __future__ import annotations++import glob+import hashlib+import importlib.metadata+import importlib.util+import json+import os+from pathlib import Path+from typing import Callable, List++from . import config+from .longbench_pro_metrics import metric_family, score, summary_components+from .scoring import strip_thinking++_MAX_TRUNCATION_PASSES = 16+++def load_examples(data_dir: str = None) -> List[dict]:+ from datasets import load_from_disk++ ds = load_from_disk(data_dir or config.LONGBENCH_PRO_DIR)+ return [dict(ex) for ex in ds]+++def build_prompt(example: dict, *, thinking: bool = False) -> str:+ """The context and the question, in the phrasing the mode calls for.++ The dataset ships the question twice. ``question_nonthinking`` asks for the+ answer directly; ``question_thinking`` is the one written for a model that+ reasons first. Reports 1 and 4 used the former because they ran the model+ non-thinking, so the field is chosen rather than fixed.+ """+ key = "question_thinking" if thinking else "question_nonthinking"+ return f"{example['context']}\n\n\n\n{example[key]}"+++def _middle_truncate(tokens: List[int], budget: int) -> List[int]:+ if len(tokens) <= budget:+ return tokens+ head = (budget + 1) // 2+ tail = budget - head+ return tokens[:head] if tail == 0 else tokens[:head] + tokens[-tail:]+++def _fit_budget(prompt, *, tokenize, detokenize, budget):+ tokens = tokenize(prompt)+ if len(tokens) <= budget:+ return prompt, tokens, False+ target = budget+ for _ in range(_MAX_TRUNCATION_PASSES):+ prompt = detokenize(_middle_truncate(tokens, target))+ tokens = tokenize(prompt)+ if len(tokens) <= budget:+ return prompt, tokens, True+ target = max(1, target - (len(tokens) - budget))+ raise RuntimeError(+ f"prompt still exceeds {budget} tokens after {_MAX_TRUNCATION_PASSES} passes"+ )+++def build(examples, *, tokenize: Callable, detokenize: Callable,+ prompt_budget=None, thinking: bool = False,+ reject_truncation: bool = False):+ """Return (conversations, prepared) for the vLLM chat runner."""+ budget = prompt_budget or config.LONGBENCH_PROMPT_BUDGET+ conversations, prepared = [], []+ for ex in examples:+ p, toks, was_truncated = _fit_budget(+ build_prompt(ex, thinking=thinking),+ tokenize=tokenize, detokenize=detokenize, budget=budget,+ )+ if reject_truncation and was_truncated:+ raise ValueError(+ f"LongBench-Pro row {ex.get('id')!r} exceeds the "+ f"{budget}-token prompt budget; truncation is forbidden"+ )+ conversations.append([{"role": "user", "content": p}])+ prepared.append({"example": ex, "prompt_tokens": len(toks),+ "prompt_was_truncated": was_truncated})+ return conversations, prepared+++def grade(prepared, completions) -> List[dict]:+ records = []+ for item, comp in zip(prepared, completions, strict=True):+ ex = item["example"]+ raw_completion = str(comp.get("text") or "")+ pred = strip_thinking(raw_completion)+ secondary = str(ex.get("secondary_task") or "")+ family = metric_family(secondary)+ value = None if family == "summary" else score(+ secondary, ex["answer"], pred+ )+ records.append({+ "id": ex.get("id"),+ "primary_task": ex.get("primary_task"),+ "secondary_task": secondary,+ "metric_family": family,+ "language": ex.get("language"),+ "token_length": ex.get("token_length"),+ "contextual_requirement": ex.get("contextual_requirement"),+ "difficulty": ex.get("difficulty"),+ "answers": ex.get("answer"),+ "prediction": pred,+ "raw_completion": raw_completion,+ "score": value,+ "f1": value if family == "f1" else None,+ "prompt_tokens": comp.get("prompt_tokens", item["prompt_tokens"]),+ "prompt_was_truncated": item["prompt_was_truncated"],+ "gen_tokens": comp.get("gen_tokens"),+ "finish_reason": comp.get("finish_reason"),+ })+ return records+++METRIC_KEY = "score"+++def _sha256(path: str) -> str:+ digest = hashlib.sha256()+ with open(path, "rb") as handle:+ for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""):+ digest.update(chunk)+ return digest.hexdigest()+++def _model_hashes(path: str) -> dict[str, str]:+ root = Path(path)+ if not root.is_dir():+ return {}+ return {+ str(candidate.relative_to(root)): _sha256(str(candidate))+ for candidate in sorted(root.rglob("*"))+ if candidate.is_file()+ }+++def _package_versions(names) -> dict[str, str]:+ versions = {}+ for name in names:+ try:+ versions[name] = importlib.metadata.version(name)+ except importlib.metadata.PackageNotFoundError:+ versions[name] = "missing"+ return versions+++class T4SummaryScorer:+ """Lazy official-compatible LongBench-Pro T4 scorer."""++ def __init__(self, embedding_model_path: str):+ from sentence_transformers import SentenceTransformer++ self.embedding_model_path = embedding_model_path+ self.embedding_model = SentenceTransformer(+ embedding_model_path,+ tokenizer_kwargs={"padding_side": "left"},+ )++ def components(self, answers, prediction, *, is_zh: bool):+ return summary_components(+ self.embedding_model, answers, prediction, is_zh=is_zh+ )+++def _load_official_utils(path: str):+ spec = importlib.util.spec_from_file_location(+ "longbench_pro_official_utils", path+ )+ if spec is None or spec.loader is None:+ raise RuntimeError(f"cannot load official evaluator utilities at {path}")+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def verify_official_summary_parity(+ scorer: T4SummaryScorer, official_utils_path: str+) -> dict:+ """Compare local and pinned official components on bilingual fixtures."""+ official = _load_official_utils(official_utils_path)+ fixtures = {+ "english": {+ "answers": [+ "A red fox crosses the river at dawn.",+ "At dawn, a red fox crosses the river.",+ "The river is crossed by a red fox at dawn.",+ ],+ "prediction": "A red fox crosses the river at dawn.",+ "is_zh": False,+ },+ "chinese": {+ "answers": [+ "清晨,一只红狐穿过河流。",+ "一只红色狐狸在清晨过河。",+ "红狐于清晨穿过了河流。",+ ],+ "prediction": "清晨,一只红狐穿过河流。",+ "is_zh": True,+ },+ }+ results = {}+ for name, fixture in fixtures.items():+ answers = official.normalize_answers(fixture["answers"])+ prediction = official.normalize_prediction_abstract(+ fixture["prediction"]+ )+ official_semantic = float(+ official.Summary_Max_Semantic_Similarity(+ scorer.embedding_model, answers, prediction+ )+ )+ official_rouge = float(+ official.Summary_Max_Rouge_L(+ answers, prediction, fixture["is_zh"]+ )+ )+ official_score = float(+ official.Summary(+ scorer.embedding_model,+ fixture["answers"],+ fixture["prediction"],+ fixture["is_zh"],+ )+ )+ local = scorer.components(+ fixture["answers"],+ fixture["prediction"],+ is_zh=fixture["is_zh"],+ )+ official_values = {+ "semantic_similarity": official_semantic,+ "rouge_l": official_rouge,+ "score": official_score,+ }+ differences = {+ key: abs(local[key] - official_values[key]) for key in local+ }+ if any(value > 1e-6 for value in differences.values()):+ raise RuntimeError(+ f"official T4 parity failed for {name}: {differences}"+ )+ results[name] = {+ "local": local,+ "official": official_values,+ "absolute_difference": differences,+ }+ return results+++def score_t4_run(+ run_dir: str,+ *,+ embedding_model_path: str,+ official_utils_path: str,+ evaluator_commit: str,+ embedding_revision: str,+) -> dict:+ """Post-score immutable sharded generations into sibling scored files."""+ source_paths = sorted(+ path+ for path in glob.glob(+ os.path.join(run_dir, "shard_*", "longbench_pro__*.jsonl")+ )+ if not path.endswith(".scored.jsonl")+ )+ if not source_paths:+ raise ValueError(f"no LongBench-Pro generation JSONL files under {run_dir}")++ source_hashes = {path: _sha256(path) for path in source_paths}+ scorer = T4SummaryScorer(embedding_model_path)+ fixture_results = verify_official_summary_parity(+ scorer, official_utils_path+ )+ total_rows = 0+ t4_rows = 0+ output_paths = []+ for source_path in source_paths:+ output_path = source_path.removesuffix(".jsonl") + ".scored.jsonl"+ temporary = f"{output_path}.tmp.{os.getpid()}"+ with open(source_path, encoding="utf-8") as source, open(+ temporary, "w", encoding="utf-8"+ ) as destination:+ for line in source:+ if not line.strip():+ continue+ row = json.loads(line)+ total_rows += 1+ if str(row.get("secondary_task", "")).startswith("T4."):+ answers = row.get("answers")+ if not isinstance(answers, list):+ raise ValueError(+ f"T4 row {row.get('id')!r} has no saved references"+ )+ values = scorer.components(+ answers,+ row.get("prediction", ""),+ is_zh=row.get("language") == "Chinese",+ )+ row["semantic_similarity"] = values["semantic_similarity"]+ row["rouge_l"] = values["rouge_l"]+ row["score"] = values["score"]+ t4_rows += 1+ destination.write(json.dumps(row, ensure_ascii=False) + "\n")+ os.replace(temporary, output_path)+ output_paths.append(output_path)++ unchanged = {path: _sha256(path) for path in source_paths}+ if unchanged != source_hashes:+ raise RuntimeError("a generation JSONL changed during T4 scoring")++ parameter = next(scorer.embedding_model.parameters(), None)+ manifest = {+ "schema": 1,+ "run_dir": os.path.realpath(run_dir),+ "source_files": source_hashes,+ "scored_files": {path: _sha256(path) for path in output_paths},+ "rows": total_rows,+ "t4_rows": t4_rows,+ "official_evaluator": {+ "commit": evaluator_commit,+ "utils_path": os.path.realpath(official_utils_path),+ "utils_sha256": _sha256(official_utils_path),+ },+ "embedding_model": {+ "path": os.path.realpath(embedding_model_path),+ "revision": embedding_revision,+ "files": _model_hashes(embedding_model_path),+ "device": str(getattr(scorer.embedding_model, "device", "unknown")),+ "dtype": str(getattr(parameter, "dtype", "unknown")),+ },+ "packages": _package_versions(+ (+ "torch",+ "transformers",+ "sentence-transformers",+ "torchao",+ "jieba",+ "rouge",+ )+ ),+ "fixture_parity": fixture_results,+ }+ manifest_path = os.path.join(run_dir, "t4_scoring_manifest.json")+ temporary = f"{manifest_path}.tmp.{os.getpid()}"+ with open(temporary, "w", encoding="utf-8") as handle:+ json.dump(manifest, handle, ensure_ascii=False, indent=2, sort_keys=True)+ handle.write("\n")+ os.replace(temporary, manifest_path)+ return manifestdiff --git a/evaluation/longbench_pro_metrics.py b/evaluation/longbench_pro_metrics.pynew file mode 100644index 0000000..82e186c--- /dev/null+++ b/evaluation/longbench_pro_metrics.py@@ -0,0 +1,169 @@+"""Task-specific metrics for LongBench-Pro, routed by secondary task.++The benchmark's answers are ordered lists and its prompts require the final+answer to follow an ``[Answer]`` / ``[答案]`` marker. The normalization here+therefore deliberately matches :mod:`evaluation.scoring`: case-fold, trim, collapse+internal whitespace, and score only the final answer area.+"""+from __future__ import annotations++import math+from collections.abc import Callable, Sequence++from .scoring import (+ F1_Score,+ normalize_answers,+ normalize_prediction,+ strip_thinking,+)+++def _lines(values: Sequence[str] | str) -> list[str]:+ if isinstance(values, str):+ return normalize_prediction(values)+ return normalize_answers(list(values))+++def substring_exact_match(answers: Sequence[str], prediction: str) -> float:+ """LongBench-Pro SubEM: fraction of gold lines exactly predicted."""+ gold = _lines(answers)+ pred = _lines(prediction)+ if not gold or not pred:+ return 0.0+ return sum(float(answer in pred) for answer in gold) / len(gold)+++def accuracy(answers: Sequence[str], prediction: str) -> float:+ """First-line exact accuracy for T3/T11 multiple-choice questions."""+ gold = _lines(answers)+ pred = _lines(prediction)+ return float(bool(gold) and bool(pred) and gold[0] == pred[0])+++def pairwise_accuracy(answers: Sequence[str], prediction: str) -> float:+ """Gold pairs in correct order, divided by all predicted-item pairs."""+ gold = _lines(answers)+ pred = _lines(prediction)+ if len(gold) < 2 or len(pred) < 2:+ return 0.0+ ranks = {value: index for index, value in enumerate(pred)}+ correct = 0+ for left_index, left in enumerate(gold):+ for right in gold[left_index + 1 :]:+ if left in ranks and right in ranks and ranks[left] < ranks[right]:+ correct += 1+ total = len(pred) * (len(pred) - 1) // 2+ return correct / total+++def ndcg_at_k(answers: Sequence[str], prediction: str) -> float:+ """TREC NDCG@k with gold rank as linear graded relevance."""+ gold = _lines(answers)+ pred = _lines(prediction)+ if not gold or not pred:+ return 0.0+ k = len(gold)+ relevance = {value: k - index for index, value in enumerate(gold)}+ run_scores = {value: len(pred) - index for index, value in enumerate(pred)}+ ranked = sorted(run_scores, key=run_scores.get, reverse=True)[:k]+ dcg = sum(+ relevance.get(value, 0) / math.log2(rank + 2.0)+ for rank, value in enumerate(ranked)+ )+ ideal = sum(+ value / math.log2(rank + 2.0)+ for rank, value in enumerate(sorted(relevance.values(), reverse=True))+ )+ return dcg / ideal if ideal else 0.0+++_F1_PREFIXES = ("T5.", "T6.2", "T7.", "T9.")+_SUBEM_PREFIXES = ("T6.1", "T8.", "T10.")+++def metric_family(secondary_task: str) -> str:+ task = str(secondary_task).strip()+ if task.startswith("T1."):+ return "ndcg"+ if task.startswith("T2.") or task.startswith("T6.3"):+ return "pairwise_accuracy"+ if task.startswith(("T3.", "T11.")):+ return "accuracy"+ if task.startswith(_F1_PREFIXES):+ return "f1"+ if task.startswith(_SUBEM_PREFIXES):+ return "subem"+ if task.startswith("T4."):+ return "summary"+ raise ValueError(f"no LongBench-Pro metric registered for {secondary_task!r}")+++_METRICS: dict[str, Callable[[Sequence[str], str], float]] = {+ "ndcg": ndcg_at_k,+ "pairwise_accuracy": pairwise_accuracy,+ "accuracy": accuracy,+ "f1": F1_Score,+ "subem": substring_exact_match,+}+++def score(secondary_task: str, answers: Sequence[str], prediction: str) -> float:+ family = metric_family(secondary_task)+ if family == "summary":+ raise ValueError("T4 summary rows must be scored after generation")+ return float(_METRICS[family](answers, prediction))+++def normalize_summary_prediction(prediction: str) -> str:+ """Official T4 normalization applied to the final answer only."""+ return " ".join(strip_thinking(str(prediction or "")).lower().strip().split())+++def summary_components(+ embedding_model,+ answers: Sequence[str],+ prediction: str,+ *,+ is_zh: bool,+ jieba_module=None,+ rouge_class=None,+) -> dict[str, float]:+ """Return the two official T4 components and their 50/50 score."""+ normalized_answers = normalize_answers(list(answers))+ normalized_prediction = normalize_summary_prediction(prediction)+ if not normalized_answers or not normalized_prediction:+ return {"semantic_similarity": 0.0, "rouge_l": 0.0, "score": 0.0}++ answer_embeddings = embedding_model.encode(normalized_answers)+ prediction_embeddings = embedding_model.encode([normalized_prediction])+ similarities = embedding_model.similarity(+ answer_embeddings, prediction_embeddings+ )+ semantic_similarity = float(similarities.max().cpu().numpy())++ rouge_answers = normalized_answers+ rouge_prediction = normalized_prediction+ if is_zh:+ if jieba_module is None:+ import jieba as jieba_module++ rouge_answers = [+ " ".join(list(jieba_module.cut(answer, cut_all=False)))+ for answer in rouge_answers+ ]+ rouge_prediction = " ".join(+ list(jieba_module.cut(rouge_prediction, cut_all=False))+ )+ if rouge_class is None:+ from rouge import Rouge as rouge_class++ scores = rouge_class().get_scores(+ [rouge_prediction] * len(rouge_answers), rouge_answers, avg=False+ )+ rouge_l = max(float(value["rouge-l"]["f"]) for value in scores)+ combined = 0.5 * semantic_similarity + 0.5 * rouge_l+ return {+ "semantic_similarity": semantic_similarity,+ "rouge_l": rouge_l,+ "score": combined,+ }diff --git a/evaluation/mrcr.py b/evaluation/mrcr.pynew file mode 100644index 0000000..009f956--- /dev/null+++ b/evaluation/mrcr.py@@ -0,0 +1,73 @@+"""OpenAI MRCR benchmark (SequenceMatcher ratio).++Uses the pre-tokenized/filtered cache selected in ``evaluation/config.py``: each row+carries its multi-turn ``messages``, gold ``answer`` and the+``random_string_to_prepend`` hash prefix. Grading is the official metric — the+response must start with the hash prefix (a single leading Qwen ``<think>``+block is stripped first), then it's the difflib ratio vs the gold answer.+"""+from __future__ import annotations++import json+from difflib import SequenceMatcher+from typing import List++from . import config+from .scoring import strip_thinking+++def _strip_think(response: str) -> str:+ return strip_thinking(response)+++def grade_one(response: str, answer: str, prefix: str) -> float:+ if response is None:+ return 0.0+ response = _strip_think(response)+ if not response.startswith(prefix):+ return 0.0+ r = response[len(prefix):]+ a = answer[len(prefix):] if answer.startswith(prefix) else answer+ return float(SequenceMatcher(None, r, a).ratio())+++def load_examples(cache_path: str = None) -> List[dict]:+ rows = []+ with open(cache_path or config.MRCR_FILTERED_CACHE) as f:+ for line in f:+ line = line.strip()+ if line:+ rows.append(json.loads(line))+ return rows+++def build(examples):+ """Return (conversations, prepared) for the vLLM chat runner."""+ conversations, prepared = [], []+ for ex in examples:+ conversations.append(ex["messages"])+ prepared.append({"example": ex, "prompt_tokens": ex.get("prompt_tokens")})+ return conversations, prepared+++def grade(prepared, completions) -> List[dict]:+ records = []+ for item, comp in zip(prepared, completions, strict=True):+ ex = item["example"]+ pred = str(comp.get("text") or "")+ prefix = ex["random_string_to_prepend"]+ stripped = _strip_think(pred)+ records.append({+ "id": ex.get("id"),+ "n_needles": ex.get("n_needles"),+ "ratio": grade_one(pred, ex["answer"], prefix),+ "ok_prefix": bool(stripped) and stripped.startswith(prefix),+ "prompt_tokens": item["prompt_tokens"],+ "prediction": pred,+ "gen_tokens": comp.get("gen_tokens"),+ "finish_reason": comp.get("finish_reason"),+ })+ return records+++METRIC_KEY = "ratio"diff --git a/evaluation/ruler.py b/evaluation/ruler.pynew file mode 100644index 0000000..9ce640e--- /dev/null+++ b/evaluation/ruler.py@@ -0,0 +1,109 @@+"""RULER needle retrieval at the registered 122,880-token stress point.++The staged rows are generated by NVIDIA/RULER at a pinned commit. We keep the+official prompts and reference lists, but intentionally do not prepend+RULER's answer prefix: Qwen3-Thinking reasons before its final answer, so the+prefix cannot occupy the position upstream designed it for. The thinking+block is removed before applying RULER's recall metric so a value considered+and rejected during reasoning does not count as retrieved.+"""+from __future__ import annotations++import json+import os+from typing import List++from . import config+from .scoring import strip_thinking+++METRIC_KEY = "recall"+TASKS = (+ "niah_single_1",+ "niah_single_2",+ "niah_single_3",+ "niah_multikey_1",+ "niah_multikey_2",+ "niah_multikey_3",+ "niah_multivalue",+ "niah_multiquery",+)+++def load_examples(+ path: str = None, *, shard_index: int = None, num_shards: int = None,+ stratified_limit: int = None,+) -> List[dict]:+ path = path or config.RULER_NIAH_PROMPTS+ if shard_index is not None:+ if num_shards is None:+ raise ValueError("num_shards is required with shard_index")+ stem, extension = path.rsplit(".", 1)+ if stratified_limit is None:+ staged_shard = (+ f"{stem}.shard_{shard_index}_of_{num_shards}.{extension}"+ )+ else:+ staged_shard = (+ f"{stem}.smoke_{stratified_limit}.shard_{shard_index}_of_"+ f"{num_shards}.{extension}"+ )+ if not os.path.isfile(staged_shard):+ raise FileNotFoundError(+ f"staged RULER shard is missing: {staged_shard}; rebuild with "+ "tools/prepare_data.py --steps ruler --force"+ )+ path = staged_shard+ rows = []+ with open(path) as handle:+ for line in handle:+ if line.strip():+ row = json.loads(line)+ if row.get("task") not in TASKS:+ raise ValueError(f"unexpected RULER task {row.get('task')!r}")+ rows.append(row)+ rows.sort(key=lambda row: row["bench_index"])+ return rows+++def build(examples):+ conversations = [+ [{"role": "user", "content": example["input"]}]+ for example in examples+ ]+ prepared = [+ {"example": example, "prompt_tokens": example.get("prompt_tokens")}+ for example in examples+ ]+ return conversations, prepared+++def recall(response: str, references: List[str]) -> float:+ """RULER's ``string_match_all`` for one row, on the final answer only."""+ answer = strip_thinking(response).lower()+ if not references:+ return 0.0+ return sum(reference.lower() in answer for reference in references) / len(references)+++def grade(prepared, completions) -> List[dict]:+ records = []+ for item, completion in zip(prepared, completions, strict=True):+ example = item["example"]+ raw = str(completion.get("text") or "")+ records.append({+ "bench_index": example["bench_index"],+ "task": example["task"],+ "source_index": example.get("source_index"),+ "references": example["outputs"],+ "recall": recall(raw, example["outputs"]),+ "prompt_tokens": completion.get(+ "prompt_tokens", item.get("prompt_tokens")+ ),+ "generator_length": example.get("length"),+ "answer_token_position": example.get("token_position_answer"),+ "raw_output": raw,+ "gen_tokens": completion.get("gen_tokens"),+ "finish_reason": completion.get("finish_reason"),+ })+ return recordsdiff --git a/evaluation/run_eval.py b/evaluation/run_eval.pynew file mode 100644index 0000000..36309fd--- /dev/null+++ b/evaluation/run_eval.py@@ -0,0 +1,1200 @@+"""Evaluate NVFP4 KV-cache quality on long-context, reasoning and code work.++For each requested KV mode a single in-process vLLM engine is built and every+benchmark is scored against it, then the engine is torn down and the next mode+loads. Results stream to per-cell JSONL and a final ``summary.md`` /+``summary.json`` table (mean per cell + delta vs the BF16 baseline).++Modes: ``bitdecoding-nvfp4`` and ``bitdecoding-mxfp4`` (CUSTOM), ``bf16``, ``fp8``,+``turboquant-4bit-nc``, ``turboquant-4bit-nc-all-layers``, and ``nvfp4``+(official vLLM NVFP4).+BitDecoding-NVFP4 automatically calibrates key smoothing and probability scaling.++ MODEL_PATH=/path/to/Qwen3-4B-Thinking-2507 \\+ python -m evaluation.run_eval --modes bitdecoding-nvfp4 --limit 20 --out-dir _out/smoke+"""+from __future__ import annotations++import argparse+import gc+import hashlib+import json+import os+import subprocess+import sys+from pathlib import Path+import time+from typing import List++from . import (+ aime,+ aime25,+ config,+ gpqa,+ lcb,+ longbench_pro,+ mrcr,+ ruler,+)++_BENCHMARKS = {+ "longbench_pro": (longbench_pro, longbench_pro.METRIC_KEY, "LongBench-Pro F1"),+ "mrcr": (mrcr, mrcr.METRIC_KEY, "MRCR ratio"),+ "aime": (aime, aime.METRIC_KEY, "AIME 2024 Avg@32"),+ "aime25": (aime25, aime25.METRIC_KEY, "AIME 2025 Avg@32"),+ "lcb": (lcb, lcb.METRIC_KEY, "LiveCodeBench v6 pass@1"),+ "ruler_niah": (ruler, ruler.METRIC_KEY, "RULER NIAH recall"),+ "gpqa": (gpqa, gpqa.METRIC_KEY, "GPQA Diamond Avg@8"),+}+++def _write_jsonl(path, records):+ os.makedirs(os.path.dirname(path), exist_ok=True)+ temporary = f"{path}.tmp.{os.getpid()}"+ with open(temporary, "w") as f:+ for r in records:+ f.write(json.dumps(r, ensure_ascii=False) + "\n")+ f.flush()+ os.fsync(f.fileno())+ os.replace(temporary, path)+++def _append_jsonl(path, records):+ """Durably append a completed scheduler group to a partial result."""+ os.makedirs(os.path.dirname(path), exist_ok=True)+ with open(path, "a") as handle:+ for record in records:+ handle.write(json.dumps(record, ensure_ascii=False) + "\n")+ handle.flush()+ os.fsync(handle.fileno())+++def _read_jsonl(path):+ records = []+ with open(path) as handle:+ for line_number, line in enumerate(handle, start=1):+ if not line.strip():+ continue+ try:+ records.append(json.loads(line))+ except json.JSONDecodeError as error:+ raise RuntimeError(+ f"invalid JSONL checkpoint {path}:{line_number}: {error}"+ ) from error+ return records+++def _partial_jsonl_path(path):+ stem, extension = os.path.splitext(path)+ return f"{stem}.partial{extension}"+++_IDENTITY_KEYS = (+ "_source_index",+ "id",+ "bench_index",+ "task",+ "problem",+ "problem_index",+ "repeat",+ "token_length",+ "secondary_task",+ "bucket",+ "needle_name",+)+++def _example_identity(example):+ return {key: example.get(key) for key in _IDENTITY_KEYS}+++def _record_identity(record):+ identity = record.get("_selection_identity")+ if not isinstance(identity, dict):+ raise RuntimeError("resumable result row lacks _selection_identity")+ return identity+++def _load_result_prefix(path, prepared):+ """Load only an exact deterministic prefix of the selected shard."""+ records = _read_jsonl(path)+ if len(records) > len(prepared):+ raise RuntimeError(+ f"checkpoint {path} has {len(records)} rows for a "+ f"{len(prepared)}-row shard"+ )+ for index, record in enumerate(records):+ expected = _example_identity(prepared[index]["example"])+ observed = _record_identity(record)+ if observed != expected:+ raise RuntimeError(+ f"checkpoint {path} row {index} identity {observed!r} "+ f"does not match selected shard {expected!r}"+ )+ return records+++def _write_json(path, value):+ os.makedirs(os.path.dirname(path), exist_ok=True)+ temporary = f"{path}.tmp.{os.getpid()}"+ with open(temporary, "w") as handle:+ json.dump(value, handle, ensure_ascii=False, indent=2, sort_keys=True)+ handle.write("\n")+ os.replace(temporary, path)+++def _write_or_validate_json(path, value):+ """Create an immutable manifest, or prove an existing one is identical."""+ if not os.path.exists(path):+ _write_json(path, value)+ return+ with open(path) as handle:+ existing = json.load(handle)+ if existing != value:+ raise RuntimeError(+ f"existing run manifest does not match the requested contract: {path}"+ )+++def _sha256(path):+ digest = hashlib.sha256()+ with open(path, "rb") as handle:+ for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""):+ digest.update(chunk)+ return digest.hexdigest()+++def _file_identity(path):+ if not isinstance(path, str) or not os.path.isfile(path):+ return None+ return {+ "path": os.path.realpath(path),+ "size": os.path.getsize(path),+ "sha256": _sha256(path),+ }+++def _selection_sha256(examples_by_benchmark):+ """Digest the rows a run selected, not merely how many it selected.++ ``bench_index`` is what identifies a row in the benchmarks that carry no+ ``id``. Without it LiveCodeBench selections hash to the same+ value for any two selections of equal size, so a shard or ``--limit``+ mismatch would pass the resume check that exists to catch exactly that.+ """+ rows = [+ {+ "benchmark": benchmark,+ "_source_index": row.get("_source_index"),+ "id": row.get("id"),+ "bench_index": row.get("bench_index"),+ "task": row.get("task"),+ "problem": row.get("problem"),+ "problem_index": row.get("problem_index"),+ "repeat": row.get("repeat"),+ "bucket": row.get("bucket"),+ "token_length": row.get("token_length"),+ "secondary_task": row.get("secondary_task"),+ }+ for benchmark in sorted(examples_by_benchmark)+ for row in examples_by_benchmark[benchmark]+ ]+ payload = json.dumps(rows, ensure_ascii=False, sort_keys=True).encode("utf-8")+ return hashlib.sha256(payload).hexdigest()+++def _default_seeds(prepared, base_seed: int):+ """One stable draw per selected row when a benchmark has no seed scheme."""+ seeds = []+ for item in prepared:+ payload = json.dumps(+ _example_identity(item["example"]),+ ensure_ascii=False,+ sort_keys=True,+ ).encode("utf-8")+ offset = int.from_bytes(hashlib.sha256(payload).digest()[:4], "little")+ seeds.append((base_seed + offset) % (2**31 - 1))+ return seeds+++def _select_explicit_rows(examples, indices):+ """Select audited source rows without renumbering their identities or seeds."""+ if not indices or any(type(i) is not int for i in indices) or len(set(indices)) != len(indices):+ raise ValueError("explicit selection must contain unique integer source indices")+ wanted = set(indices)+ selected = [row for row in examples if row["_source_index"] in wanted]+ if len(selected) != len(indices):+ raise ValueError("explicit selection references rows outside the selected protocol")+ return selected+++def _frozen_prompt_audit(name, tokenizer, conversations, prepared, model_path, thinking):+ """Verify a schema-2 prompt freeze using the exact serving tokenizer."""+ path = os.environ.get("BITDECODING_LONG_CONTEXT_TOKEN_AUDIT")+ if not path:+ return None+ with open(path) as handle:+ audit = json.load(handle)+ if audit.get("schema") != 2:+ return None+ rows = {json.dumps(row["identity"], sort_keys=True): row+ for row in audit["rows"] if row["benchmark"] == name}+ selected = []+ for messages, item in zip(conversations, prepared, strict=True):+ expected = rows.get(json.dumps(_example_identity(item["example"]), sort_keys=True))+ rendered = tokenizer.apply_chat_template(messages, tokenize=False,+ add_generation_prompt=True, **config.chat_template_kwargs(model_path, thinking))+ tokens = tokenizer.encode(rendered, add_special_tokens=False)+ token_hash = hashlib.sha256(json.dumps(tokens, separators=(",", ":")).encode()).hexdigest()+ if expected is None or expected["token_sha256"] != token_hash:+ raise RuntimeError("prompt token IDs differ from the frozen evaluation input")+ if expected["rendered_sha256"] != hashlib.sha256(rendered.encode()).hexdigest():+ raise RuntimeError("rendered prompt differs from the frozen evaluation input")+ item["prompt_tokens"] = len(tokens)+ selected.append(expected)+ return selected++def _run_manifest(+ args, mode, mode_config, runner, examples_cache, selection_metadata+):+ audit_path = os.environ.get("BITDECODING_LONG_CONTEXT_TOKEN_AUDIT")+ engine = runner.engine_contract()+ if (mode == "bitdecoding-nvfp4" and getattr(args, "calibration_report", None)+ and config.RECIPE["underflow_aware_probability_scaling"]["enabled"]):+ tau = config.RECIPE["underflow_aware_probability_scaling"]["tau"]+ if engine.get("nvfp4_p_scale_tau") != tau:+ raise RuntimeError("the scoring engine did not load the calibrated tau")+ requested_rope = engine.get("requested_rope_scaling")+ resolved_rope = engine.get("resolved_rope_scaling")+ if requested_rope is not None:+ if not isinstance(resolved_rope, dict):+ raise RuntimeError("engine did not expose the requested YaRN config")+ requested_semantic = {+ "rope_type": requested_rope.get("rope_type", requested_rope.get("type")),+ "factor": float(requested_rope.get("factor")),+ "original_max_position_embeddings": int(+ requested_rope.get("original_max_position_embeddings")+ ),+ }+ resolved_semantic = {+ "rope_type": resolved_rope.get("rope_type", resolved_rope.get("type")),+ "factor": float(resolved_rope.get("factor")),+ "original_max_position_embeddings": int(+ resolved_rope.get("original_max_position_embeddings")+ ),+ }+ if resolved_semantic != requested_semantic:+ raise RuntimeError(+ f"resolved rope config {resolved_semantic} does not match "+ f"requested {requested_semantic}"+ )+ resolved_max_model_len = engine.get("resolved_max_model_len")+ if resolved_max_model_len is not None and resolved_max_model_len != args.max_model_len:+ raise RuntimeError(+ f"engine resolved max_model_len={resolved_max_model_len}, "+ f"requested {args.max_model_len}"+ )+ expected_cache_dtype = str(mode_config["kv_cache_dtype"])+ if engine.get("resolved_cache_dtype") != expected_cache_dtype:+ raise RuntimeError(+ f"engine resolved cache dtype {engine.get('resolved_cache_dtype')!r}, "+ f"but arm {mode!r} requires {expected_cache_dtype!r}"+ )+ layer_dtypes = engine.get("layer_cache_dtype_counts") or {}+ layer_backends = engine.get("layer_attention_backend_counts") or {}+ if not layer_dtypes or not layer_backends:+ raise RuntimeError(+ "engine did not expose per-layer KV dtype/backend attestation"+ )+ if mode in {"turboquant-4bit-nc", "turboquant-4bit-nc-all-layers"}:+ all_layers = bool(mode_config.get("turboquant_all_layers"))+ layer_count = _model_layer_count(args.model_path)+ expected_skip = [] if all_layers else [+ str(index) for index in (0, 1, layer_count - 2, layer_count - 1)+ ]+ if engine.get("kv_cache_dtype_skip_layers") != expected_skip:+ raise RuntimeError(+ "TurboQuant skip layers differ from the registered arm: "+ f"{engine.get('kv_cache_dtype_skip_layers')} != {expected_skip}"+ )+ expected_layer_dtypes = (+ {expected_cache_dtype: layer_count}+ if all_layers+ else {"auto": 4, expected_cache_dtype: layer_count - 4}+ )+ if layer_dtypes != expected_layer_dtypes:+ raise RuntimeError(+ f"TurboQuant layer KV dtypes are {layer_dtypes}, expected "+ f"{expected_layer_dtypes}"+ )+ expected_turboquant_layers = layer_count if all_layers else layer_count - 4+ expected_native_layers = 0 if all_layers else 4+ if (+ layer_backends.get("TURBOQUANT") != expected_turboquant_layers+ or sum(+ count for backend, count in layer_backends.items()+ if backend != "TURBOQUANT"+ ) != expected_native_layers+ ):+ raise RuntimeError(+ "TurboQuant resolved the wrong compressed/native layer split: "+ f"expected {expected_turboquant_layers}/"+ f"{expected_native_layers}, got {layer_backends}"+ )+ else:+ expected_layers = {expected_cache_dtype: sum(layer_dtypes.values())}+ if layer_dtypes != expected_layers:+ raise RuntimeError(+ f"arm {mode!r} resolved mixed layer KV dtypes {layer_dtypes}"+ )+ expected_backend = (+ "CUSTOM"+ if mode_config.get("custom_backend")+ else (mode_config.get("engine_kwargs") or {}).get(+ "attention_backend"+ )+ )+ if expected_backend is not None and layer_backends != {+ expected_backend: sum(layer_backends.values())+ }:+ raise RuntimeError(+ f"arm {mode!r} requires backend {expected_backend}, got "+ f"{layer_backends}"+ )+ return {+ "schema": 2,+ "mode": mode,+ "mode_label": mode_config["label"],+ "model": os.path.realpath(args.model_path),+ "model_layer_count": (+ _model_layer_count(args.model_path)+ if os.path.isfile(os.path.join(args.model_path, "config.json"))+ else None+ ),+ "model_revision": os.environ.get("BITDECODING_MODEL_REVISION"),+ "data_dir": os.path.realpath(args.data_dir),+ "dataset_manifest": _file_identity(+ os.path.join(args.data_dir, "MANIFEST.json")+ ),+ "ruler_generator": _file_identity(+ os.path.join(args.data_dir, "ruler", "generator.json")+ ),+ "benchmarks": list(args.benchmarks),+ "shard_index": args.shard_index,+ "num_shards": args.num_shards,+ "max_model_len": args.max_model_len,+ "gpu_memory_utilization": args.gpu_memory_utilization,+ "scale_mode_override": getattr(args, "scale_mode", None),+ "thinking": args.thinking,+ "seed": args.seed,+ "sampling": {+ name: _sampling_for(+ name, args.thinking, mode, max_model_len=args.max_model_len,+ model_path=args.model_path,+ )+ for name in args.benchmarks+ },+ "selection_sha256": _selection_sha256(examples_cache),+ "selected_rows": {+ name: len(rows) for name, rows in examples_cache.items()+ },+ "selection_metadata": selection_metadata,+ "prompt_budget": int(os.environ.get(+ "BITDECODING_LONGBENCH_PROMPT_BUDGET", config.LONGBENCH_PROMPT_BUDGET+ )),+ "engine": engine,+ "recipe": config.RECIPE if mode_config.get("custom_backend") else None,+ "calibration": _file_identity(getattr(args, "calibration_report", None)),+ "mode_config": mode_config,+ "sidecars": {+ key: _file_identity(mode_config.get(key))+ for key in ("key_shift",)+ if mode_config.get(key) is not None+ },+ "token_audit": _file_identity(audit_path),+ "resumable_output": {+ "schema": 1,+ "unit": "completed_scheduler_group",+ "source_files": {+ name: _file_identity(os.path.join(os.path.dirname(__file__), name))+ for name in (+ "config.py", "gpqa.py", "longbench_pro.py", "ruler.py",+ "run_eval.py", "vllm_runner.py", "aime.py", "aime25.py",+ "lcb.py", "mrcr.py", "scoring.py", "longbench_pro_metrics.py",+ )+ },+ },+ }+++def _mean(records, key):+ vals = [float(r[key]) for r in records if r.get(key) is not None]+ return (sum(vals) / len(vals)) if vals else None+++def _validate_shard(shard_index: int, num_shards: int) -> None:+ if num_shards <= 0:+ raise ValueError("num_shards must be positive")+ if not 0 <= shard_index < num_shards:+ raise ValueError(+ f"shard_index must be in [0, {num_shards}), got {shard_index}"+ )+++def _select_shard(examples, *, shard_index: int, num_shards: int):+ _validate_shard(shard_index, num_shards)+ return examples[shard_index::num_shards]+++def _evenly_spaced(examples, limit: int):+ """Select deterministic coverage across an ordered benchmark."""+ if limit <= 0:+ raise ValueError("limit must be positive")+ if len(examples) <= limit:+ return list(examples)+ if limit == 1:+ return [examples[0]]+ return [+ examples[index * (len(examples) - 1) // (limit - 1)]+ for index in range(limit)+ ]+++def _stratified_slice(name: str, examples, limit: int):+ """Select smoke rows spanning long-context length/problem coverage."""+ if name == "longbench_pro":+ buckets = {}+ for example in examples:+ buckets.setdefault(str(example.get("token_length")), []).append(example)+ ordered = sorted(+ buckets,+ key=lambda value: int(value[:-1]) if value.endswith("k") else value,+ )+ base, remainder = divmod(min(limit, len(examples)), len(ordered))+ selected = []+ for position, bucket in enumerate(ordered):+ take = base + (position < remainder)+ selected.extend(_evenly_spaced(buckets[bucket], take))+ # The official T4 scorer gate needs one English and one Chinese row,+ # while keeping the exact per-bucket smoke allocation unchanged.+ for language in ("English", "Chinese"):+ if any(+ str(row.get("secondary_task", "")).startswith("T4.")+ and row.get("language") == language+ for row in selected+ ):+ continue+ candidate = next(+ (+ row for row in examples+ if str(row.get("secondary_task", "")).startswith("T4.")+ and row.get("language") == language+ ),+ None,+ )+ if candidate is None:+ continue+ bucket = str(candidate.get("token_length"))+ replace_at = next(+ (+ index for index, row in enumerate(selected)+ if str(row.get("token_length")) == bucket+ and not str(row.get("secondary_task", "")).startswith("T4.")+ ),+ None,+ )+ if replace_at is not None:+ selected[replace_at] = candidate+ return selected+ if name == "mrcr":+ ordered = sorted(examples, key=lambda row: int(row["prompt_tokens"]))+ return _evenly_spaced(ordered, limit)+ if name == "ruler_niah":+ tasks = {}+ for example in examples:+ tasks.setdefault(example["task"], []).append(example)+ ordered = [task for task in ruler.TASKS if task in tasks]+ base, remainder = divmod(min(limit, len(examples)), len(ordered))+ selected = []+ for position, task in enumerate(ordered):+ selected.extend(+ _evenly_spaced(tasks[task], base + (position < remainder))+ )+ return selected+ return _evenly_spaced(examples, limit)+++def _long_context_filter(name: str, examples, modes):+ if name != "longbench_pro":+ return examples+ examples = [+ row for row in examples+ if not str(row.get("secondary_task", "")).startswith("T4.")+ ]+ included_prefixes = tuple(+ value.strip() for value in+ os.environ.get("BITDECODING_LONG_CONTEXT_LBP_TASK_PREFIXES", "").split(",")+ if value.strip()+ )+ if included_prefixes:+ examples = [+ row for row in examples+ if str(row.get("secondary_task", "")).startswith(included_prefixes)+ ]+ excluded = {+ value.strip() for value in+ os.environ.get(+ "BITDECODING_LONG_CONTEXT_EXCLUDED_LBP_BUCKETS",+ ",".join(config.EVALUATION["longbench_excluded_buckets"]),+ ).split(",")+ if value.strip()+ }+ if not excluded:+ return examples+ return [row for row in examples if str(row.get("token_length")) not in excluded]+++def build_parser():+ ap = argparse.ArgumentParser(description=__doc__,+ formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--model-path", default=config.MODEL_PATH)+ ap.add_argument("--data-dir", default=config.DATA_DIR)+ ap.add_argument("--selection-file", default=None,+ help="JSON mapping benchmark names to frozen source-row indices")+ ap.add_argument("--out-dir", default="_out")+ ap.add_argument("--modes", nargs="+", default=["bitdecoding-nvfp4"],+ choices=sorted(config.KV_MODES),+ help="bitdecoding-nvfp4 and bitdecoding-mxfp4 use CUSTOM; bf16, fp8, "+ "both turboquant-4bit-nc arms and official nvfp4 are "+ "baselines")+ ap.add_argument("--benchmarks", nargs="+", default=list(_BENCHMARKS),+ choices=list(_BENCHMARKS))+ ap.add_argument("--limit", type=int, default=None,+ help="cap examples per benchmark (smoke runs)")+ ap.add_argument("--stratified-limit", type=int, default=None,+ help="cap examples while spanning long-context length/problem coverage")+ ap.add_argument(+ "--resume", action="store_true",+ help="resume an exact partial shard at scheduler-group boundaries",+ )+ ap.add_argument("--shard-index", type=int, default=0,+ help="zero-based deterministic example shard")+ ap.add_argument("--num-shards", type=int, default=1,+ help="number of deterministic example shards")+ ap.add_argument("--tp-size", type=int, default=1)+ ap.add_argument("--distributed-executor-backend", choices=("mp", "ray"))+ ap.add_argument("--gpu-memory-utilization", type=float, default=0.9)+ ap.add_argument("--max-model-len", type=int, default=config.MAX_MODEL_LEN)+ ap.add_argument("--enforce-eager", dest="enforce_eager",+ action="store_true", default=None)+ ap.add_argument("--no-enforce-eager", dest="enforce_eager",+ action="store_false")+ ap.add_argument("--attention-backend", default=None,+ help="e.g. FLASHINFER (nvfp4/SM100) or TRITON_ATTN")+ ap.add_argument("--scale-mode", default=None,+ help="CUSTOM NVFP4 block-scale candidates: a mode name "+ "such as four_six for all three tensors, or "+ "key=...,value=...,query=... to set them apart")+ ap.add_argument("--seed", type=int, default=42)+ ap.add_argument("--thinking", action="store_true",+ help="run every benchmark in Qwen3 thinking mode, which "+ "also switches LongBench-Pro and MRCR off greedy "+ "(see config.THINKING_BENCH_OVERRIDES)")+ ap.add_argument("--score-longbench-t4", action="store_true",+ help="post-score saved LongBench-Pro T4 generations in --out-dir")+ ap.add_argument("--embedding-model-path", default=None)+ ap.add_argument("--official-longbench-utils", default=None)+ ap.add_argument(+ "--official-evaluator-commit",+ default="a1d3fd6eab275981a75f03ebda2169b72b7b876e",+ )+ ap.add_argument(+ "--embedding-model-revision",+ default="1d8ad4ca9b3dd8059ad90a75d4983776a23d44af",+ )+ return ap+++def _model_family(model_path=None):+ return config.model_family(model_path)+++def _model_layer_count(model_path):+ try:+ with open(os.path.join(model_path, "config.json")) as handle:+ model_config = json.load(handle)+ except (OSError, TypeError, json.JSONDecodeError) as error:+ raise RuntimeError(+ f"cannot read model config for layer attestation: {model_path}"+ ) from error+ for candidate in (model_config, model_config.get("text_config", {})):+ for key in ("num_hidden_layers", "num_layers", "n_layer"):+ if candidate.get(key) is not None:+ return int(candidate[key])+ raise RuntimeError(f"model config has no layer count: {model_path}")+++def _sampling_for(name, thinking: bool, mode: str, max_model_len=None, model_path=None) -> dict:+ defaults = dict(config.BENCH_DEFAULTS[name])+ if thinking:+ defaults.update(config.THINKING_BENCH_OVERRIDES.get(name, {}))+ window = config.SHORT_CONTEXT_MAX_MODEL_LEN if max_model_len is None else max_model_len+ if window > config.SHORT_CONTEXT_MAX_MODEL_LEN:+ defaults.update(config.LONG_CONTEXT_BENCH_OVERRIDES.get(name, {}))+ if thinking:+ family = _model_family(model_path)+ defaults.update(config.MODEL_THINKING_SAMPLING[family])+ if family == "glm":+ defaults.pop("top_k", None)+ return defaults+++def _budgets(defaults, prepared, max_model_len):+ """The generation budget: one number, or one per row filling the window."""+ want = defaults["max_new_tokens"]+ if want != config.FILL_WINDOW:+ return want+ budgets = []+ for item in prepared:+ prompt_tokens = item.get("prompt_tokens")+ if prompt_tokens is None:+ raise ValueError(+ "a benchmark asking to fill the window has to report "+ "prompt_tokens for every row"+ )+ budgets.append(+ max_model_len - prompt_tokens - config.GENERATION_MARGIN+ )+ if min(budgets) <= 0:+ raise ValueError(+ f"a prompt leaves no room to answer in a {max_model_len} "+ f"window (smallest budget {min(budgets)})"+ )+ return budgets+++def _templated_prompt_lengths(tokenizer, conversations, thinking, model_path=None):+ """Count the exact chat-template tokens used by a long-window engine."""+ lengths = []+ for messages in conversations:+ kwargs = {+ "tokenize": True,+ "add_generation_prompt": True,+ **config.chat_template_kwargs(model_path, thinking),+ }+ try:+ value = tokenizer.apply_chat_template(messages, **kwargs)+ except TypeError:+ kwargs.pop("enable_thinking", None)+ value = tokenizer.apply_chat_template(messages, **kwargs)+ # ``apply_chat_template(tokenize=True)`` may return a Transformers+ # BatchEncoding. ``len(value)`` then counts fields rather than token+ # IDs, typically producing 2 and giving fill-window generation an+ # invalid budget. Unwrap the actual input-id sequence first.+ if hasattr(value, "input_ids"):+ value = value.input_ids+ elif isinstance(value, dict) and "input_ids" in value:+ value = value["input_ids"]+ if hasattr(value, "tolist"):+ value = value.tolist()+ if value and isinstance(value[0], list):+ value = value[0]+ lengths.append(len(value))+ return lengths+++def _verify_frozen_completions(audited_rows, completions):+ if audited_rows is None:+ return+ for expected, completion in zip(audited_rows, completions, strict=True):+ if (completion.get("prompt_tokens") != expected["prompt_tokens"]+ or completion.get("prompt_token_sha256") != expected["token_sha256"]):+ raise RuntimeError("engine consumed prompt token IDs that differ from the input freeze")+++def _run_benchmark(name, runner, examples_cache, args, mode, *, output_path=None):+ module, metric_key, _label = _BENCHMARKS[name]+ thinking = args.thinking+ defaults = _sampling_for(+ name, args.thinking, mode, max_model_len=args.max_model_len,+ model_path=getattr(args, "model_path", None),+ )++ if name == "longbench_pro":+ conversations, prepared = module.build(+ examples_cache[name], tokenize=runner.tokenize,+ detokenize=runner.detokenize, thinking=thinking,+ reject_truncation=False,+ )+ else:+ conversations, prepared = module.build(examples_cache[name])++ # GLM's tokenizer is not Qwen's tokenizer. The staged RULER/MRCR+ # ``prompt_tokens`` are Qwen-side measurements, so using them to fill a+ # GLM window would silently give GLM the wrong answer budget. Count the+ # exact template that the serving tokenizer will consume after the engine+ # is built and retain those lengths in the manifest/results.+ if (+ name in {"ruler_niah", "mrcr"}+ and _model_family(getattr(args, "model_path", None)) == "glm"+ ):+ for item, prompt_tokens in zip(+ prepared,+ _templated_prompt_lengths(runner.tokenizer, conversations, thinking,+ getattr(args, "model_path", None)),+ strict=True,+ ):+ item["prompt_tokens"] = prompt_tokens+ frozen = _frozen_prompt_audit(name, getattr(runner, "tokenizer", None), conversations, prepared,+ getattr(args, "model_path", None), thinking)++ # A benchmark that samples the same prompt more than once has to say so;+ # one seed for such a batch makes the repeats identical.+ seed = (+ module.seeds(prepared, args.seed)+ if hasattr(module, "seeds")+ else _default_seeds(prepared, args.seed)+ )+ budget = _budgets(defaults, prepared, args.max_model_len)+ if frozen is not None:+ for i, row in enumerate(frozen):+ row_budget = budget[i] if isinstance(budget, list) else budget+ if row["sampling_seed"] != seed[i] or row["generation_budget"] != row_budget:+ raise RuntimeError("sampling seed or generation budget differs from the freeze")+ def attach_identity(records, selected_prepared, selected_seeds, selected_completions):+ row_seeds = (+ list(selected_seeds)+ if isinstance(selected_seeds, (list, tuple))+ else [selected_seeds] * len(selected_prepared)+ )+ for record, item, row_seed, completion in zip(+ records, selected_prepared, row_seeds, selected_completions, strict=True+ ):+ record["_selection_identity"] = _example_identity(item["example"])+ record["_sampling_seed"] = int(row_seed)+ if "output_token_ids" in completion:+ record["output_token_ids"] = completion["output_token_ids"]+ if "prompt_token_sha256" in completion:+ record["prompt_token_sha256"] = completion["prompt_token_sha256"]+ return records++ resumable = args.resume and output_path is not None+ if resumable:+ partial_path = _partial_jsonl_path(output_path)+ if os.path.exists(output_path):+ if os.path.exists(partial_path):+ raise RuntimeError(+ f"both final and partial results exist: {output_path}, "+ f"{partial_path}"+ )+ records = _load_result_prefix(output_path, prepared)+ if len(records) != len(prepared):+ raise RuntimeError(+ f"final result {output_path} has {len(records)}/"+ f"{len(prepared)} rows"+ )+ print(f" [resume] {name}: final result already complete", flush=True)+ return records, metric_key++ records = records if os.path.exists(output_path) else (+ _load_result_prefix(partial_path, prepared)+ if os.path.exists(partial_path) else []+ )+ completed = len(records)+ if completed:+ print(+ f" [resume] {name}: validated {completed}/{len(prepared)} "+ f"checkpoint rows",+ flush=True,+ )+ batch_size = int(+ getattr(runner, "_request_batch_size", None)+ or len(conversations) or 1+ )++ def selected(value, start, stop):+ return value[start:stop] if isinstance(value, (list, tuple)) else value++ for start in range(completed, len(prepared), batch_size):+ stop = min(start + batch_size, len(prepared))+ completions = runner.chat(+ conversations[start:stop],+ temperature=defaults["temperature"], top_p=defaults["top_p"],+ top_k=defaults.get("top_k"), min_p=defaults.get("min_p"),+ max_new_tokens=selected(budget, start, stop),+ seed=selected(seed, start, stop),+ enable_thinking=True if args.thinking else None,+ )+ _verify_frozen_completions(frozen[start:stop] if frozen is not None else None, completions)+ batch_records = attach_identity(+ module.grade(prepared[start:stop], completions),+ prepared[start:stop],+ selected(seed, start, stop),+ completions,+ )+ if len(batch_records) != stop - start:+ raise RuntimeError(+ f"{name} graded {len(batch_records)} rows for a "+ f"{stop - start}-row scheduler group"+ )+ _append_jsonl(partial_path, batch_records)+ records.extend(batch_records)+ print(+ f" [checkpoint] {name}: {len(records)}/{len(prepared)} rows",+ flush=True,+ )+ if os.path.exists(partial_path):+ os.replace(partial_path, output_path)+ elif not os.path.exists(output_path):+ _write_jsonl(output_path, [])+ else:+ completions = runner.chat(+ conversations,+ temperature=defaults["temperature"], top_p=defaults["top_p"],+ top_k=defaults.get("top_k"), min_p=defaults.get("min_p"),+ max_new_tokens=budget, seed=seed,+ enable_thinking=True if args.thinking else None,+ )+ _verify_frozen_completions(frozen, completions)+ records = attach_identity(module.grade(prepared, completions), prepared, seed, completions)+ truncated = sum(1 for r in records if r.get("finish_reason") == "length")+ if truncated:+ # Loud because it is the failure mode that looks like a result: every+ # arm truncating at once reads as "the cache does not matter here".+ described = (f"{min(budget)}..{max(budget)}"+ if isinstance(budget, list) else budget)+ print(+ f" [truncation] {name}: {truncated}/{len(records)} hit "+ f"max_new_tokens={described}",+ flush=True,+ )+ return records, metric_key+++def _prepare_calibration(args, scale_mode=None):+ """Fit the key mean and tau before the scoring process imports the kernel."""+ modes = set(args.modes) & {"bitdecoding-nvfp4", "bitdecoding-mxfp4"}+ if not modes:+ return+ fit_tau = "bitdecoding-nvfp4" in modes and config.needs_tau_calibration(config.RECIPE)+ smooth = config.RECIPE["outlier_aware_key_quantization"]["key_smoothing"]+ if not fit_tau and (not smooth or all(+ os.path.isfile(config.KV_MODES[mode].get("key_shift") or "") for mode in modes+ )):+ return++ directory = Path(args.out_dir).resolve() / "calibration"+ sidecar = directory / "key_smoothing.safetensors"+ report_path = sidecar.with_suffix(".json")+ mode = config.KV_MODES["bitdecoding-nvfp4" if fit_tau else sorted(modes)[0]]+ scales = scale_mode if scale_mode is not None else mode["scale_mode"]+ additional = {+ "nvfp4_scale_mode": scales, "nvfp4_query_terms": mode["query_terms"],+ "nvfp4_rotation": mode["rotation"], "nvfp4_key_smoothing": None,+ }+ expected = {+ "model_path": os.path.realpath(args.model_path),+ "max_model_len": args.max_model_len, "context_tokens": args.max_model_len - 1,+ "tensor_parallel_size": args.tp_size, "additional_config": additional,+ }+ if not report_path.exists():+ if sidecar.exists():+ raise ValueError(f"incomplete calibration at {sidecar}; choose a new output directory")+ directory.mkdir(parents=True, exist_ok=True)+ command = [+ sys.executable, str(Path(config.REPO_ROOT) / "tools/calibrate_key_smooth_sidecar.py"),+ "--model-path", args.model_path, "--output-path", str(sidecar),+ "--max-model-len", str(args.max_model_len),+ "--context-tokens", str(args.max_model_len - 1),+ "--tensor-parallel-size", str(args.tp_size), "--max-num-seqs", "1",+ "--gpu-memory-utilization", str(args.gpu_memory_utilization),+ "--scale-mode", json.dumps(scales),+ "--query-terms", str(mode["query_terms"]), "--rotation", mode["rotation"],+ ]+ environment = os.environ.copy()+ if fit_tau:+ command += ["--probability-multiplier", str(config.RECIPE["underflow_aware_probability_scaling"]["c"])]+ probability = config.probability_environment(config.RECIPE, calibration=True)+ else:+ probability = {"NVFP4_P_SCALE_MODE": None, "NVFP4_P_SCALE_TAU": None}+ for key, value in {**probability, "NVFP4_P_SCALE_DIVISOR": None}.items():+ if value is None:+ environment.pop(key, None)+ else:+ environment[key] = value+ environment["PYTHONPATH"] = os.pathsep.join([+ str(Path(config.REPO_ROOT) / "src"), config.REPO_ROOT,+ environment.get("PYTHONPATH", ""),+ ])+ print("Calibrating key smoothing and probability scaling before evaluation...", flush=True)+ subprocess.run(command, env=environment, check=True)++ report = json.loads(report_path.read_text())+ if not sidecar.is_file() or any(report.get(key) != value for key, value in expected.items()):+ raise ValueError(f"calibration does not match this model and recipe: {report_path}")+ if fit_tau:+ probability = report.get("probability_scaling", {})+ resolved = {**config.RECIPE, "underflow_aware_probability_scaling": {+ **config.RECIPE["underflow_aware_probability_scaling"], "tau": probability.get("tau"),+ }}+ config.probability_environment(resolved)+ if probability.get("c") != config.RECIPE["underflow_aware_probability_scaling"]["c"]:+ raise ValueError(f"calibration probability multiplier differs from the recipe: {report_path}")+ config.RECIPE["underflow_aware_probability_scaling"]["tau"] = probability["tau"]+ print(f"Calibrated tau={probability['tau']:.9g}", flush=True)+ if smooth:+ for name in modes:+ config.KV_MODES[name]["key_shift"] = str(sidecar)+ args.calibration_report = str(report_path)+++def _configure_probability_scaling(mode):+ if mode == "bitdecoding-nvfp4":+ environment = config.probability_environment(config.RECIPE)+ elif mode == "bitdecoding-mxfp4":+ environment = {"NVFP4_P_SCALE_MODE": None, "NVFP4_P_SCALE_TAU": None}+ else:+ return+ environment["NVFP4_P_SCALE_DIVISOR"] = None+ for name, value in environment.items():+ if value is None:+ os.environ.pop(name, None)+ else:+ os.environ[name] = value+++def main():+ args = build_parser().parse_args()+ if args.score_longbench_t4:+ if not args.embedding_model_path or not args.official_longbench_utils:+ raise SystemExit(+ "--score-longbench-t4 requires --embedding-model-path and "+ "--official-longbench-utils"+ )+ manifest = longbench_pro.score_t4_run(+ args.out_dir,+ embedding_model_path=args.embedding_model_path,+ official_utils_path=args.official_longbench_utils,+ evaluator_commit=args.official_evaluator_commit,+ embedding_revision=args.embedding_model_revision,+ )+ print(+ f"[T4] scored {manifest['t4_rows']}/{manifest['rows']} rows "+ f"under {args.out_dir}",+ flush=True,+ )+ return+ try:+ _validate_shard(args.shard_index, args.num_shards)+ except ValueError as error:+ raise SystemExit(str(error)) from error+ if args.limit is not None and args.stratified_limit is not None:+ raise SystemExit("--limit and --stratified-limit are mutually exclusive")+ if args.stratified_limit is not None and args.stratified_limit <= 0:+ raise SystemExit("--stratified-limit must be positive")+ scale_mode = None+ if args.scale_mode is not None:+ from nvfp4_decode_kernel.scale_mode import parse_scale_mode_spec++ try:+ scale_mode = parse_scale_mode_spec(args.scale_mode)+ except ValueError as error:+ raise SystemExit(str(error)) from error+ config.MODEL_PATH = args.model_path+ config.configure_data_dir(args.data_dir)++ problems = config.verify_prestaging(args.benchmarks)+ problems += config.verify_dependencies(args.benchmarks)+ if problems:+ raise SystemExit("Pre-staging incomplete:\n- " + "\n- ".join(problems))+ _prepare_calibration(args, scale_mode)+ problems = config.verify_sidecars(args.modes)+ if problems:+ raise SystemExit("Calibration incomplete:\n- " + "\n- ".join(problems))+ if "bitdecoding-nvfp4" in args.modes:+ _configure_probability_scaling("bitdecoding-nvfp4")++ # Load each dataset once (shared across modes).+ examples_cache = {}+ selection_metadata = {}+ for name in args.benchmarks:+ module = _BENCHMARKS[name][0]+ pre_sharded = False+ if (+ name == "ruler_niah"+ and args.limit is None+ and not args.selection_file+ ):+ ex = module.load_examples(+ shard_index=args.shard_index,+ num_shards=args.num_shards,+ stratified_limit=args.stratified_limit,+ )+ pre_sharded = True+ else:+ ex = module.load_examples()+ for source_index, row in enumerate(ex):+ row.setdefault("_source_index", source_index)+ ex = _long_context_filter(name, ex, args.modes)+ if args.selection_file:+ if args.limit or args.stratified_limit or pre_sharded:+ raise ValueError("explicit selection cannot be combined with a limit or pre-sharded data")+ with open(args.selection_file) as handle:+ indices = json.load(handle)[name]+ ex = _select_explicit_rows(ex, indices)+ selection_metadata.setdefault(name, {})["selection_file"] = _file_identity(args.selection_file)+ if args.limit:+ ex = ex[: args.limit]+ elif args.stratified_limit and not pre_sharded:+ ex = _stratified_slice(name, ex, args.stratified_limit)+ if not pre_sharded:+ ex = _select_shard(+ ex,+ shard_index=args.shard_index,+ num_shards=args.num_shards,+ )+ examples_cache[name] = ex+ print(+ f"[data] {name}: {len(ex)} examples "+ f"(shard {args.shard_index}/{args.num_shards})",+ flush=True,+ )++ results = {} # (benchmark, mode) -> (mean, n)+ for mode in args.modes:+ _configure_probability_scaling(mode)+ mode_config = config.resolve_mode(mode, args.model_path)+ kv_dtype = mode_config["kv_cache_dtype"]+ print(f"\n=== KV mode: {mode} ({mode_config['label']}) "+ f"kv_cache_dtype={kv_dtype} ===", flush=True)+ from .vllm_runner import VllmRunner++ engine_started = time.monotonic()+ runner = VllmRunner(+ model_path=args.model_path, kv_cache_dtype=mode_config,+ max_model_len=args.max_model_len,+ gpu_memory_utilization=args.gpu_memory_utilization,+ tensor_parallel_size=args.tp_size,+ distributed_executor_backend=args.distributed_executor_backend,+ enforce_eager=args.enforce_eager,+ attention_backend=args.attention_backend,+ scale_mode=scale_mode,+ )+ _append_jsonl(os.path.join(args.out_dir, f"engine_startup__{mode}.jsonl"), [{+ "mode": mode, "benchmarks": args.benchmarks,+ "host": os.uname().nodename, "pid": os.getpid(),+ "started_unix": time.time() - (time.monotonic() - engine_started),+ "engine_startup_seconds": time.monotonic() - engine_started,+ "includes_tokenizer_load_and_graph_initialization": True,+ "model_path": os.path.realpath(args.model_path),+ "tensor_parallel_size": args.tp_size,+ }])+ try:+ manifest_name = (+ "manifest__" + "_".join(args.benchmarks) + f"__{mode}.json"+ )+ _write_or_validate_json(+ os.path.join(args.out_dir, manifest_name),+ _run_manifest(+ args,+ mode,+ mode_config,+ runner,+ examples_cache,+ selection_metadata,+ ),+ )+ for name in args.benchmarks:+ output_path = os.path.join(+ args.out_dir, f"{name}__{mode}.jsonl"+ )+ health_path = output_path.removesuffix(".jsonl") + ".health.json"+ preserve_health = os.path.isfile(output_path) and os.path.isfile(+ health_path+ )+ try:+ import torch++ torch.cuda.reset_peak_memory_stats()+ except (ImportError, RuntimeError):+ torch = None+ started = time.monotonic()+ records, metric_key = _run_benchmark(+ name, runner, examples_cache, args, mode,+ output_path=output_path,+ )+ if not args.resume:+ _write_jsonl(output_path, records)+ if not preserve_health:+ max_gpu_memory = (+ torch.cuda.max_memory_reserved() / 2**20+ if torch is not None and torch.cuda.is_available()+ else None+ )+ elapsed = time.monotonic() - started+ _write_json(health_path, {+ "schema": 1,+ "benchmark": name,+ "mode": mode,+ "n": len(records),+ "wall_clock_s": elapsed,+ "max_gpu_mem_mib": max_gpu_memory,+ "generated_tokens": sum(+ int(record.get("gen_tokens") or 0)+ for record in records+ ),+ "finished_on_length": sum(+ record.get("finish_reason") == "length"+ for record in records+ ),+ })+ mean = _mean(records, metric_key)+ results[(name, mode)] = (mean, len(records))+ mean_str = "n/a (deferred)" if mean is None else f"{mean:.4f}"+ print(f" {name}: mean {metric_key} = {mean_str} (n={len(records)})",+ flush=True)+ finally:+ runner.shutdown()+ del runner+ gc.collect()++ _write_summary(args, results)+++def _write_summary(args, results):+ benchmarks = args.benchmarks+ header = ["KV mode"]+ for b in benchmarks:+ header += [b, f"{b} Δ vs bf16"]+ lines = ["| " + " | ".join(header) + " |",+ "|" + "|".join(["---"] * len(header)) + "|"]+ base = {b: (results.get((b, "bf16")) or (None, 0))[0] for b in benchmarks}+ summary_json = {"results": {}}+ for mode in args.modes:+ row = [config.KV_MODES[mode]["label"]]+ for b in benchmarks:+ mean, n = results.get((b, mode), (None, 0))+ summary_json["results"][f"{b}__{mode}"] = {"mean": mean, "n": n}+ if mean is None:+ row += ["—", "—"]+ continue+ row.append(f"{mean:.4f}")+ bm = base.get(b)+ row.append(f"{mean - bm:+.4f}" if bm is not None else "—")+ lines.append("| " + " | ".join(row) + " |")+ table = "\n".join(lines)++ os.makedirs(args.out_dir, exist_ok=True)+ md_path = os.path.join(args.out_dir, "summary.md")+ with open(md_path, "w") as f:+ f.write("# BitDecoding-FP4 — NVFP4 KV-cache evaluation\n\n")+ f.write(f"Model: `{args.model_path}`\n\n")+ f.write(table + "\n")+ with open(os.path.join(args.out_dir, "summary.json"), "w") as f:+ json.dump(summary_json, f, indent=2)+ print("\n" + table)+ print(f"\nWrote {md_path}")+++if __name__ == "__main__":+ main()diff --git a/evaluation/scoring.py b/evaluation/scoring.pynew file mode 100644index 0000000..5c6ec81--- /dev/null+++ b/evaluation/scoring.py@@ -0,0 +1,70 @@+"""F1 scoring for LongBench-Pro T9.++Vendored (trimmed to the four public names used here) from+caskcsg/longcontext/LongBench-Pro/modules/utils.py, preserving its scoring+semantics.+"""+from __future__ import annotations++import re+from typing import List, Set+++_THINK_RE = re.compile(r"^\s*<think>.*?</think>\s*", flags=re.DOTALL)+++def strip_thinking(text: str) -> str:+ """Remove one leading Qwen reasoning block.++ Some chat templates include ``<think>`` in the assistant prefill, so the+ returned generation starts with the reasoning body and only contains the+ closing ``</think>`` tag. Accept both that form and the self-contained+ ``<think>...</think>`` form.+ """+ value = str(text or "")+ stripped = _THINK_RE.sub("", value, count=1)+ if stripped == value and "</think>" in value:+ stripped = value.split("</think>", 1)[1].lstrip()+ return stripped+++def get_answer_area(text: str) -> str:+ text = strip_thinking(text)+ if "[Answer]" in text or "[答案]" in text:+ if "[Answer]" in text:+ last = text.rfind("[Answer]")+ if last != -1:+ text = text[last + len("[Answer]"):]+ else:+ last = text.rfind("[答案]")+ if last != -1:+ text = text[last + len("[答案]"):]+ return text.strip()+++def _fix_space(text: str) -> str:+ return " ".join(text.split())+++def normalize_answers(answers: List[str]) -> List[str]:+ return [_fix_space(a.lower().strip()) for a in answers]+++def normalize_prediction(prediction: str) -> List[str]:+ area = get_answer_area(prediction).lower()+ return [_fix_space(line.strip()) for line in area.split("\n")]+++def F1_Score(answers: List[str], prediction: str) -> float:+ a = normalize_answers(answers)+ p = normalize_prediction(prediction)+ a_set: Set[str] = set(a)+ p_set: Set[str] = set(p)+ common = a_set & p_set+ if not common or not p_set or not a_set:+ return 0.0+ precision = len(common) / len(p_set)+ recall = len(common) / len(a_set)+ if precision + recall == 0:+ return 0.0+ return (2 * precision * recall) / (precision + recall)diff --git a/evaluation/vllm_runner.py b/evaluation/vllm_runner.pynew file mode 100644index 0000000..c8b39a3--- /dev/null+++ b/evaluation/vllm_runner.py@@ -0,0 +1,614 @@+"""In-process vLLM engine wrapper for the NVFP4 KV-cache evaluation.++One :class:`VllmRunner` == one loaded model with a fixed ``kv_cache_dtype``.+The official vLLM NVFP4 KV cache is selected purely via+``kv_cache_dtype="nvfp4"`` (served by FlashInfer's trtllm-gen path); the BF16+baseline uses ``kv_cache_dtype="auto"``. The repository CUSTOM backend is+selected by a bound mode in :mod:`evaluation.config`, which carries every+scheduler/cache argument that backend requires.++A mode that sets ``custom_backend`` is built through+:func:`nvfp4_vllm.build_llm` rather than through ``LLM`` directly. That+function owns the five settings the write path cannot work without and refuses+to be contradicted, which is the difference between an engine that fails to+start and one that starts and quietly serves from a different cache.+"""+from __future__ import annotations++import copy+import hashlib+import json+from collections import Counter+from pathlib import Path+from typing import List++from . import config+++def _path_sha256(path: Path) -> str:+ digest = hashlib.sha256()+ files = [path] if path.is_file() else sorted(+ child for child in path.rglob("*") if child.is_file()+ )+ for child in files:+ relative = child.name if path.is_file() else child.relative_to(path).as_posix()+ digest.update(relative.encode())+ digest.update(b"\0")+ with child.open("rb") as handle:+ for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""):+ digest.update(chunk)+ return digest.hexdigest()+++class VllmRunner:+ def __init__(self, model_path=None, *, kv_cache_dtype="auto",+ max_model_len=None, gpu_memory_utilization=0.9,+ tensor_parallel_size=1, enforce_eager=None, distributed_executor_backend=None,+ attention_backend=None, max_num_seqs=None, block_size=None,+ enable_prefix_caching=None, enable_chunked_prefill=None,+ max_num_batched_tokens=None, pipeline_parallel_size=None,+ scale_mode=None, rope_scaling=None,+ logits_processors=None):+ from transformers import AutoTokenizer++ self.model_path = model_path or config.MODEL_PATH+ self.tensor_parallel_size = tensor_parallel_size+ self.tokenizer = AutoTokenizer.from_pretrained(+ self.model_path, trust_remote_code=True+ )+ if isinstance(kv_cache_dtype, dict):+ mode = dict(kv_cache_dtype)+ else:+ mode = {"kv_cache_dtype": str(kv_cache_dtype)}+ mode_kwargs = dict(mode.get("engine_kwargs", {}))+ kv_cache_dtype = str(mode["kv_cache_dtype"])+ self.kv_cache_dtype = kv_cache_dtype+ custom_backend = bool(mode.get("custom_backend"))+ mode_rope_scaling = mode.get("rope_scaling")+ if rope_scaling is not None and mode_rope_scaling is not None:+ if rope_scaling != mode_rope_scaling:+ raise ValueError("caller rope_scaling contradicts the named mode")+ selected_rope_scaling = self._validate_rope_scaling(+ mode_rope_scaling if mode_rope_scaling is not None else rope_scaling+ )++ self._check_sink(mode.get("sink_tokens"), custom_backend)++ if enforce_eager is None:+ enforce_eager = bool(mode_kwargs.get("enforce_eager", False))++ llm_kwargs = dict(+ model=self.model_path,+ dtype="bfloat16",+ kv_cache_dtype=kv_cache_dtype,+ max_model_len=max_model_len or config.MAX_MODEL_LEN,+ tensor_parallel_size=tensor_parallel_size,+ gpu_memory_utilization=gpu_memory_utilization,+ enforce_eager=enforce_eager,+ trust_remote_code=True,+ )+ if logits_processors is not None:+ llm_kwargs["logits_processors"] = logits_processors+ if selected_rope_scaling is not None:+ llm_kwargs["hf_overrides"] = {+ "rope_scaling": selected_rope_scaling,+ }+ optional_engine_kwargs = {+ "distributed_executor_backend": distributed_executor_backend,+ "attention_backend": attention_backend,+ "max_num_seqs": max_num_seqs,+ "block_size": block_size,+ "enable_prefix_caching": enable_prefix_caching,+ "enable_chunked_prefill": enable_chunked_prefill,+ "max_num_batched_tokens": max_num_batched_tokens,+ "pipeline_parallel_size": pipeline_parallel_size,+ }+ llm_kwargs.update(+ {key: value for key, value in optional_engine_kwargs.items()+ if value is not None}+ )+ if mode_kwargs:+ # A mode's requirements win over generic caller overrides. In+ # particular, a CUSTOM mode must never be detached from CUSTOM.+ # Only unchunked prefill needs the entire prompt in one batch.+ if not mode_kwargs.get("enable_chunked_prefill", False):+ mode_kwargs["max_num_batched_tokens"] = max(+ mode_kwargs["max_num_batched_tokens"],+ llm_kwargs["max_model_len"],+ )+ llm_kwargs.update(mode_kwargs)++ # Submit at most one scheduler-width of requests at a time. The+ # CUSTOM backend supports pure prefill and pure decode batches, but on+ # SM103 the bundled FA4 prefill path can abort when a heterogeneous+ # replacement prompt is JIT-compiled into an active decode batch.+ # Use the same grouping for every cache mode.+ self._request_batch_size = llm_kwargs.get("max_num_seqs")++ if custom_backend:+ self.llm = self._build_custom(llm_kwargs, mode, scale_mode)+ else:+ llm_kwargs.update(+ self._custom_backend_kwargs(+ llm_kwargs, scale_mode=scale_mode+ )+ )+ self.llm = self._build_standard(llm_kwargs, mode)++ self.requested_rope_scaling = selected_rope_scaling+++ @staticmethod+ def _validate_rope_scaling(value):+ if value is None:+ return None+ if not isinstance(value, dict):+ raise ValueError("rope_scaling must be a dictionary")+ result = copy.deepcopy(value)+ rope_type = result.get("rope_type", result.get("type"))+ if rope_type != "yarn":+ raise ValueError("only a static YaRN rope_scaling override is supported")+ factor = result.get("factor")+ original = result.get("original_max_position_embeddings")+ if not isinstance(factor, (int, float)) or factor <= 1:+ raise ValueError("YaRN factor must be greater than one")+ if not isinstance(original, int) or original <= 0:+ raise ValueError(+ "YaRN original_max_position_embeddings must be positive"+ )+ result.pop("type", None)+ result["rope_type"] = "yarn"+ return result++ def engine_contract(self) -> dict:+ """Requested and resolved positional settings for run manifests."""+ from nvfp4_decode_kernel.p_scale import (+ P_SMALL_GROUP_AMAX,+ p_scale_is_amax2_or_amax6,+ resolve_p_scale_environment,+ )+ p_scale_mode, p_scale_divisor, p_scale_grid_search = resolve_p_scale_environment()+ p_scale_tau = P_SMALL_GROUP_AMAX+ resolved = None+ max_model_len = None+ cache_dtype = None+ kv_cache_memory_bytes = None+ cache_skip_layers = None+ configured_backend = None+ block_size = None+ max_num_seqs = None+ max_num_batched_tokens = None+ enable_prefix_caching = None+ enable_chunked_prefill = None+ enable_flashinfer_autotune = None+ additional_config = None+ layer_cache_dtypes = Counter()+ layer_attention_backends = Counter()+ worker_layer_contracts = None+ worker_layer_contract_error = None+ resolved_executor_backend = None+ resolved_tp_size = self.tensor_parallel_size+ try:+ vllm_config = self.llm.llm_engine.vllm_config+ model_config = vllm_config.model_config+ parallel = getattr(vllm_config, "parallel_config", None)+ if parallel is not None:+ resolved_executor_backend = getattr(parallel, "distributed_executor_backend", None)+ resolved_tp_size = parallel.tensor_parallel_size+ max_model_len = int(model_config.max_model_len)+ hf_config = model_config.hf_config+ resolved = getattr(hf_config, "rope_scaling", None)+ if resolved is None:+ resolved = getattr(hf_config, "rope_parameters", None)+ cache = vllm_config.cache_config+ kv_cache_memory_bytes = getattr(cache, "kv_cache_memory_bytes", None)+ cache_dtype = str(cache.cache_dtype)+ cache_skip_layers = list(getattr(cache, "kv_cache_dtype_skip_layers", []) or [])+ block_size = int(cache.block_size)+ enable_prefix_caching = bool(cache.enable_prefix_caching)+ scheduler = vllm_config.scheduler_config+ max_num_seqs = int(scheduler.max_num_seqs)+ max_num_batched_tokens = int(scheduler.max_num_batched_tokens)+ enable_chunked_prefill = bool(scheduler.enable_chunked_prefill)+ enable_flashinfer_autotune = bool(+ getattr(vllm_config.kernel_config, "enable_flashinfer_autotune", False)+ )+ selected = getattr(vllm_config.attention_config, "backend", None)+ additional_config = copy.deepcopy(+ getattr(vllm_config, "additional_config", None)+ )+ configured_backend = (+ getattr(selected, "value", None) or str(selected)+ if selected is not None else None+ )+ except AttributeError:+ pass+ try:+ core = self.llm.llm_engine.engine_core+ while hasattr(core, "engine_core"):+ core = core.engine_core+ model = core.model_executor.driver_worker.worker.model_runner.model+ for module in model.modules():+ layer_dtype = getattr(module, "kv_cache_dtype", None)+ backend = getattr(module, "attn_backend", None)+ if layer_dtype is None or backend is None:+ continue+ layer_cache_dtypes[str(layer_dtype)] += 1+ layer_attention_backends[str(backend.get_name())] += 1+ except (AttributeError, TypeError):+ pass+ if not layer_cache_dtypes or not layer_attention_backends:+ try:+ worker_layer_contracts = self.llm.collective_rpc(+ "nvfp4_layer_cache_contract"+ )+ if not worker_layer_contracts:+ raise RuntimeError("worker RPC returned no contracts")+ canonical = worker_layer_contracts[0]+ if any(item != canonical for item in worker_layer_contracts[1:]):+ raise RuntimeError(+ "tensor-parallel workers reported different layer contracts: "+ f"{worker_layer_contracts}"+ )+ layer_cache_dtypes.clear()+ layer_attention_backends.clear()+ layer_cache_dtypes.update(+ canonical.get("layer_cache_dtype_counts") or {}+ )+ layer_attention_backends.update(+ canonical.get("layer_attention_backend_counts") or {}+ )+ except Exception as error:+ worker_layer_contract_error = (+ f"{type(error).__name__}: {error}"+ )+ probability_format = None+ probability_mode = None+ if layer_attention_backends.get("CUSTOM"):+ from nvfp4_decode_kernel.scale_mode import scale_format++ mode = (additional_config or {}).get("nvfp4_scale_mode", "amax6")+ if isinstance(mode, dict):+ mode = mode["key"]+ probability_format = scale_format(mode)+ probability_mode = (+ "adjacent_sse_mass" if probability_format == "mxfp4" else p_scale_mode+ )+ is_mxfp4 = probability_format == "mxfp4"+ return {+ "requested_rope_scaling": copy.deepcopy(self.requested_rope_scaling),+ "resolved_rope_scaling": copy.deepcopy(resolved),+ "resolved_max_model_len": max_model_len,+ "tensor_parallel_size": resolved_tp_size,+ "distributed_executor_backend": resolved_executor_backend,+ "resolved_cache_dtype": cache_dtype,+ **({"kv_cache_memory_bytes": kv_cache_memory_bytes} if kv_cache_memory_bytes is not None else {}),+ "kv_cache_dtype_skip_layers": cache_skip_layers,+ "configured_attention_backend": configured_backend,+ "layer_cache_dtype_counts": dict(layer_cache_dtypes),+ "layer_attention_backend_counts": dict(layer_attention_backends),+ "worker_layer_contracts": worker_layer_contracts,+ "worker_layer_contract_error": worker_layer_contract_error,+ "block_size": block_size,+ "max_num_seqs": max_num_seqs,+ "max_num_batched_tokens": max_num_batched_tokens,+ "enable_prefix_caching": enable_prefix_caching,+ "enable_chunked_prefill": enable_chunked_prefill,+ "enable_flashinfer_autotune": enable_flashinfer_autotune,+ "probability_scale_format": probability_format,+ "probability_scale_mode": probability_mode,+ "nvfp4_p_rescale_factor": (None if probability_format != "nvfp4" else 8 if p_scale_mode == "amax2_or_amax6_c8" else 1),+ "nvfp4_p_two_level": (probability_format == "nvfp4" and p_scale_mode == "two_level"),+ "nvfp4_p_scale_mode": None if is_mxfp4 else p_scale_mode,+ "nvfp4_p_scale_tau": None if is_mxfp4 else p_scale_tau,+ "nvfp4_p_scale_divisor": None if is_mxfp4 else p_scale_divisor,+ "nvfp4_p_scale_grid_search": None if is_mxfp4 else p_scale_grid_search,+ "nvfp4_p_scale_amax2_or_amax6": (+ None if is_mxfp4 else p_scale_is_amax2_or_amax6(p_scale_mode)+ ),+ "additional_config": additional_config,+ }++ def shutdown(self) -> str | None:+ """Release the vLLM engine, including a multiprocess EngineCore."""+ engine = getattr(self.llm, "llm_engine", None)+ # The shutdown entry point may be on the engine or its core client.+ for name, owner in (+ ("llm", self.llm),+ ("llm_engine", engine),+ ("llm_engine.engine_core", getattr(engine, "engine_core", None)),+ ):+ shutdown = getattr(owner, "shutdown", None)+ if callable(shutdown):+ shutdown()+ return name+ return None++ @staticmethod+ def _check_sink(expected, custom_backend: bool) -> None:+ """Check that the loaded kernel's sink matches the evaluation mode."""+ if expected is None:+ return+ if not custom_backend:+ raise ValueError("sink_tokens is a property of the CUSTOM backend")+ from nvfp4_decode_kernel.interface import SINK_TOKENS+ import nvfp4_decode_kernel++ if SINK_TOKENS != expected:+ raise SystemExit(+ f"this arm needs SINK_TOKENS={expected}, but the kernel on "+ f"the path has {SINK_TOKENS} "+ f"({nvfp4_decode_kernel.__file__}). Point PYTHONPATH at the "+ "checkout that matches the arm."+ )++ @staticmethod+ def _build_custom(llm_kwargs, mode, scale_mode):+ """The CUSTOM backend, through the constructor that owns its contract."""+ from nvfp4_vllm import build_llm++ engine = dict(llm_kwargs)+ model = engine.pop("model")+ # build_llm sets attention_config; vLLM refuses to be told both.+ engine.pop("attention_backend", None)+ engine.pop("kv_cache_dtype", None)++ requested = VllmRunner._custom_additional_config(mode, scale_mode)+ if requested:+ print(f"[custom backend] additional_config={requested}", flush=True)+ additional = dict(engine.get("additional_config") or {})+ additional.update(requested)+ engine["additional_config"] = additional+ additional = dict(engine.get("additional_config") or {})+ additional["nvfp4_key_smoothing"] = mode.get("key_shift")+ return build_llm(model, **{**engine, "additional_config": additional})++ @staticmethod+ def _build_standard(llm_kwargs, mode, *, worker_extension_cls=None):+ """Build vLLM, optionally suppressing its TurboQuant boundary skip.++ Upstream vLLM automatically keeps the first and last two transformer+ layers on native KV cache for every TurboQuant dtype. The all-layers+ evaluation arm deliberately removes that policy for the duration of+ engine construction. Restoring the static method in ``finally`` keeps+ a later boundary-protected engine in the same Python process honest.+ The run manifest independently refuses the arm unless all loaded+ layers attest TurboQuant cache and the TURBOQUANT backend.++ A benchmark can supply a worker extension for timing synchronization;+ evaluation retains the calibration and cache-inspection extension.+ """+ from vllm import LLM++ from nvfp4_vllm.calibrate import WORKER_EXTENSION_CLS++ expected_extension = worker_extension_cls or WORKER_EXTENSION_CLS+ supplied_extension = llm_kwargs.get("worker_extension_cls")+ if supplied_extension not in (None, expected_extension):+ raise ValueError(+ "standard evaluation requires worker_extension_cls="+ f"{expected_extension}, got {supplied_extension}"+ )+ llm_kwargs["worker_extension_cls"] = expected_extension++ if not mode.get("turboquant_all_layers"):+ return LLM(**llm_kwargs)+ if mode.get("kv_cache_dtype") != "turboquant_4bit_nc":+ raise ValueError(+ "turboquant_all_layers requires kv_cache_dtype="+ "'turboquant_4bit_nc'"+ )+ from vllm.model_executor.layers.quantization.turboquant.config import (+ TurboQuantConfig,+ )+ from vllm.v1.attention.backends.turboquant_attn import (+ TurboQuantAttentionBackend,+ )++ # At the pinned vLLM a TurboQuant layer's KV spec reports no quant+ # mode, so the runner shapes its cache as a skipped layer's, asking+ # the TurboQuant backend for cache dtype "auto", which it refuses;+ # Qwen3-30B reaches that in the CUDA-graph memory profile. With no+ # layer skipped, "auto" can only mean the preset. Both patches live+ # in this process, so the engine must run in it too+ # (VLLM_ENABLE_V1_MULTIPROCESSING=0).+ preset = mode["kv_cache_dtype"]+ boundary = TurboQuantConfig.__dict__["get_boundary_skip_layers"]+ cache_shape = TurboQuantAttentionBackend.__dict__["get_kv_cache_shape"]++ def preset_cache_shape(+ num_blocks, block_size, num_kv_heads, head_size, cache_dtype_str=preset+ ):+ if cache_dtype_str == "auto":+ cache_dtype_str = preset+ return cache_shape.__func__(+ num_blocks, block_size, num_kv_heads, head_size, cache_dtype_str+ )++ TurboQuantConfig.get_boundary_skip_layers = staticmethod(+ lambda model_config, n=2: []+ )+ TurboQuantAttentionBackend.get_kv_cache_shape = staticmethod(preset_cache_shape)+ try:+ return LLM(**llm_kwargs)+ finally:+ TurboQuantConfig.get_boundary_skip_layers = boundary+ TurboQuantAttentionBackend.get_kv_cache_shape = cache_shape++ @staticmethod+ def _custom_additional_config(mode, scale_mode):+ """Translate a named evaluation arm into the backend's public keys."""+ selected_scale = (+ scale_mode if scale_mode is not None else mode.get("scale_mode")+ )+ requested = {+ "nvfp4_scale_mode": selected_scale,+ "nvfp4_key_scale_geometry": mode.get("key_scale_geometry"),+ "nvfp4_query_terms": mode.get("query_terms"),+ "nvfp4_rotation": mode.get("rotation"),+ }+ return {key: value for key, value in requested.items() if value is not None}++ @staticmethod+ def _custom_backend_kwargs(llm_kwargs, *, scale_mode):+ """Apply block-scale overrides only to the CUSTOM backend."""+ if scale_mode is None:+ return {}+ if llm_kwargs.get("attention_backend") != "CUSTOM":+ raise ValueError(+ "--scale-mode is an option of the CUSTOM "+ "NVFP4 backend; pass --modes bitdecoding-nvfp4"+ )+ return {"additional_config": {"nvfp4_scale_mode": scale_mode}}++ def tokenize(self, s: str) -> List[int]:+ return self.tokenizer.encode(s, add_special_tokens=False)++ def detokenize(self, ids: List[int]) -> str:+ return self.tokenizer.decode(ids, skip_special_tokens=True)++ def chat(self, conversations, *, temperature=0.0, top_p=1.0, top_k=None,+ min_p=None, max_new_tokens=1024, seed=None,+ enable_thinking=None) -> List[dict]:+ """Run one chat completion per conversation. Returns per-request dicts+ with ``text``, ``prompt_tokens``, ``gen_tokens`` and ``finish_reason``.++ ``seed`` may be one value for the batch, or one per conversation. The+ per-conversation form is for benchmarks that sample the same prompt+ more than once: under a shared seed those repeats decode identically,+ which looks like a valid run and silently measures Avg@1.++ ``max_new_tokens`` takes the same two forms. The per-row form lets a+ long-context benchmark give each row whatever the window has left+ after its own prompt, instead of every row sharing a budget that has+ to be small enough for the longest one.++ ``enable_thinking`` is passed to the chat template. Qwen3 defaults to+ thinking, so this is stated rather than relied upon: the default is+ a property of the checkpoint's template and could change under us.+ """+ from vllm import SamplingParams++ extra = {}+ if top_k is not None:+ extra["top_k"] = top_k+ if min_p is not None:+ extra["min_p"] = min_p++ def params(one_seed, one_budget):+ return SamplingParams(+ temperature=temperature, top_p=top_p,+ max_tokens=one_budget, seed=one_seed, **extra,+ )++ def spread(value, what):+ if not isinstance(value, (list, tuple)):+ return [value] * len(conversations), False+ if len(value) != len(conversations):+ raise ValueError(+ f"got {len(value)} {what} for {len(conversations)} "+ "conversations"+ )+ return list(value), True++ seeds, per_row_seed = spread(seed, "seeds")+ budgets, per_row_budget = spread(max_new_tokens, "token budgets")+ if per_row_seed or per_row_budget:+ sp = [params(s, b) for s, b in zip(seeds, budgets)]+ else:+ sp = params(seed, max_new_tokens)+ chat_kwargs = {}+ if enable_thinking is not None:+ chat_kwargs["chat_template_kwargs"] = config.chat_template_kwargs(+ getattr(self, "model_path", None), enable_thinking+ )+ batch_size = getattr(self, "_request_batch_size", None)+ batch_size = int(batch_size or len(conversations) or 1)+ outputs = []+ for start in range(0, len(conversations), batch_size):+ stop = min(start + batch_size, len(conversations))+ batch_params = sp[start:stop] if isinstance(sp, list) else sp+ outputs.extend(+ self.llm.chat(+ conversations[start:stop], batch_params,+ use_tqdm=True, **chat_kwargs,+ )+ )+ results = []+ for out in outputs:+ comp = out.outputs[0]+ results.append({+ "text": comp.text,+ "prompt_tokens": len(out.prompt_token_ids),+ "prompt_token_sha256": hashlib.sha256(json.dumps(+ list(out.prompt_token_ids), separators=(",", ":")+ ).encode()).hexdigest(),+ "gen_tokens": len(comp.token_ids),+ "output_token_ids": list(comp.token_ids),+ "finish_reason": comp.finish_reason,+ })+ return results++ def generate(self, prompts, *, temperature=0.0, top_p=1.0, top_k=None,+ min_p=None, max_new_tokens=1024, seed=None) -> List[dict]:+ """Run raw text completions without applying a chat template.++ Seed and token budget accept the same scalar-or-per-row forms as+ :meth:`chat`.+ """+ from vllm import SamplingParams++ extra = {}+ if top_k is not None:+ extra["top_k"] = top_k+ if min_p is not None:+ extra["min_p"] = min_p++ def params(one_seed, one_budget):+ return SamplingParams(+ temperature=temperature, top_p=top_p,+ max_tokens=one_budget, seed=one_seed, **extra,+ )++ def spread(value, what):+ if not isinstance(value, (list, tuple)):+ return [value] * len(prompts), False+ if len(value) != len(prompts):+ raise ValueError(+ f"got {len(value)} {what} for {len(prompts)} prompts"+ )+ return list(value), True++ seeds, per_row_seed = spread(seed, "seeds")+ budgets, per_row_budget = spread(max_new_tokens, "token budgets")+ if per_row_seed or per_row_budget:+ sp = [params(s, b) for s, b in zip(seeds, budgets)]+ else:+ sp = params(seed, max_new_tokens)++ batch_size = getattr(self, "_request_batch_size", None)+ batch_size = int(batch_size or len(prompts) or 1)+ outputs = []+ for start in range(0, len(prompts), batch_size):+ stop = min(start + batch_size, len(prompts))+ batch_params = sp[start:stop] if isinstance(sp, list) else sp+ outputs.extend(+ self.llm.generate(+ prompts[start:stop], batch_params, use_tqdm=True,+ )+ )++ results = []+ for out in outputs:+ comp = out.outputs[0]+ results.append({+ "text": comp.text,+ "prompt_tokens": len(out.prompt_token_ids),+ "prompt_token_sha256": hashlib.sha256(json.dumps(+ list(out.prompt_token_ids), separators=(",", ":")+ ).encode()).hexdigest(),+ "gen_tokens": len(comp.token_ids),+ "output_token_ids": list(comp.token_ids),+ "finish_reason": comp.finish_reason,+ })+ return resultsdiff --git a/imgs/accuracy.svg b/imgs/accuracy.svgnew file mode 100644index 0000000..9d832d1--- /dev/null+++ b/imgs/accuracy.svg@@ -0,0 +1,3746 @@+<?xml version="1.0" encoding="utf-8" standalone="no"?>+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">+<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="1152pt" height="446.4pt" viewBox="0 0 1152 446.4" xmlns="http://www.w3.org/2000/svg" version="1.1">+<desc>{"source": "Paper accuracy table", "columns": ["Model", "Method", "LB-Pro", "MRCR", "RULER", "AIME24", "AIME25", "LCB v6", "GPQA", "Avg."], "models": {"Qwen3-4B-Thinking-2507": [{"method": "BF16 KV", "scores": ["53.27", "38.82", "96.28", "83.44", "81.15", "53.86", "66.98", "67.69"]}, {"method": "FP8 KV", "scores": ["52.60", "37.67", "94.42", "82.29", "78.75", "50.86", "66.86", "66.21"]}, {"method": "TurboQuant-4bit-nc", "scores": ["49.78", "32.91", "93.15", "77.40", "72.29", "47.43", "64.96", "62.56"]}, {"method": "SAW-INT4", "scores": ["49.92", "31.57", "91.19", "79.69", "75.63", "52.86", "64.02", "63.55"]}, {"method": "TRT-LLM-NVFP4", "scores": ["44.89", "28.84", "80.04", "76.56", "72.29", "48.71", "63.01", "59.19"]}, {"method": "BitDecoding-MXFP4", "scores": ["52.96", "35.61", "94.46", "81.56", "80.42", "55.14", "66.73", "66.70"]}, {"method": "BitDecoding-NVFP4", "scores": ["52.34", "36.68", "95.68", "82.50", "80.62", "56.14", "66.92", "67.27"]}], "Qwen3-30B-A3B-Thinking-2507": [{"method": "BF16 KV", "scores": ["60.83", "43.41", "98.39", "90.62", "83.54", "64.00", "71.28", "73.15"]}, {"method": "FP8 KV", "scores": ["58.45", "40.60", "98.03", "91.56", "82.81", "60.86", "70.58", "71.84"]}, {"method": "TurboQuant-4bit-nc", "scores": ["59.56", "42.22", "98.02", "88.65", "83.85", "56.57", "70.64", "71.36"]}, {"method": "SAW-INT4", "scores": ["57.27", "36.31", "97.58", "88.85", "83.75", "57.71", "70.39", "70.27"]}, {"method": "TRT-LLM-NVFP4", "scores": ["50.72", "28.41", "89.42", "88.54", "81.77", "61.00", "70.90", "67.25"]}, {"method": "BitDecoding-MXFP4", "scores": ["58.53", "42.71", "98.33", "91.04", "84.58", "62.29", "70.71", "72.60"]}, {"method": "BitDecoding-NVFP4", "scores": ["58.86", "42.22", "98.58", "90.83", "85.31", "62.57", "70.83", "72.74"]}]}}</desc>+ <title>Accuracy table from the paper</title>+ <metadata>+ <rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">+ <cc:Work>+ <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>+ <dc:title>Accuracy table from the paper</dc:title>+ <dc:description>Accuracy (%) for seven KV-cache methods across seven benchmarks on Qwen3-4B-Thinking-2507 and Qwen3-30B-A3B-Thinking-2507.</dc:description>+ <dc:format>image/svg+xml</dc:format>+ <dc:creator>+ <cc:Agent>+ <dc:title>BitDecoding-FP4</dc:title>+ </cc:Agent>+ </dc:creator>+ </cc:Work>+ </rdf:RDF>+ </metadata>+ <defs>+ <style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style>+ </defs>+ <g id="figure_1">+ <g id="patch_1">+ <path d="M 0 446.4+L 1152 446.4+L 1152 0+L 0 0+z+" style="fill: #ffffff"/>+ </g>+ <g id="axes_1">+ <g id="patch_2">+ <path d="M 189.36 184.32+L 1112.4 184.32+L 1112.4 206.64+L 189.36 206.64+z+" clip-path="url(#pb7bde0cdf1)" style="fill: #edf4e9"/>+ </g>+ <g id="patch_3">+ <path d="M 189.36 206.64+L 1112.4 206.64+L 1112.4 228.96+L 189.36 228.96+z+" clip-path="url(#pb7bde0cdf1)" style="fill: #edf4e9"/>+ </g>+ <g id="patch_4">+ <path d="M 189.36 340.56+L 1112.4 340.56+L 1112.4 362.88+L 189.36 362.88+z+" clip-path="url(#pb7bde0cdf1)" style="fill: #edf4e9"/>+ </g>+ <g id="patch_5">+ <path d="M 189.36 362.88+L 1112.4 362.88+L 1112.4 385.2+L 189.36 385.2+z+" clip-path="url(#pb7bde0cdf1)" style="fill: #edf4e9"/>+ </g>+ <g id="line2d_1">+ <path d="M 39.6 14.4+L 1112.4 14.4+" clip-path="url(#pb7bde0cdf1)" style="fill: none; stroke: #222222; stroke-width: 1.4"/>+ </g>+ <g id="line2d_2">+ <path d="M 401.04 41.04+L 657.36 41.04+" clip-path="url(#pb7bde0cdf1)" style="fill: none; stroke: #777777; stroke-width: 0.7"/>+ </g>+ <g id="line2d_3">+ <path d="M 668.88 41.04+L 835.92 41.04+" clip-path="url(#pb7bde0cdf1)" style="fill: none; stroke: #777777; stroke-width: 0.7"/>+ </g>+ <g id="line2d_4">+ <path d="M 847.44 41.04+L 925.2 41.04+" clip-path="url(#pb7bde0cdf1)" style="fill: none; stroke: #777777; stroke-width: 0.7"/>+ </g>+ <g id="line2d_5">+ <path d="M 936.72 41.04+L 1014.48 41.04+" clip-path="url(#pb7bde0cdf1)" style="fill: none; stroke: #777777; stroke-width: 0.7"/>+ </g>+ <g id="line2d_6">+ <path d="M 39.6 72+L 1112.4 72+" clip-path="url(#pb7bde0cdf1)" style="fill: none; stroke: #222222; stroke-width: 1.1"/>+ </g>+ <g id="line2d_7">+ <path d="M 189.36 117.36+L 1112.4 117.36+" clip-path="url(#pb7bde0cdf1)" style="fill: none; stroke: #b8b8b8; stroke-width: 0.55"/>+ </g>+ <g id="line2d_8">+ <path d="M 39.6 228.96+L 1112.4 228.96+" clip-path="url(#pb7bde0cdf1)" style="fill: none; stroke: #555555"/>+ </g>+ <g id="line2d_9">+ <path d="M 189.36 273.6+L 1112.4 273.6+" clip-path="url(#pb7bde0cdf1)" style="fill: none; stroke: #b8b8b8; stroke-width: 0.55"/>+ </g>+ <g id="line2d_10">+ <path d="M 39.6 385.2+L 1112.4 385.2+" clip-path="url(#pb7bde0cdf1)" style="fill: none; stroke: #222222; stroke-width: 1.4"/>+ </g>+ <g id="text_1">+ <!-- Long context -->+ <g style="fill: #222222" transform="translate(482.34825 33.991875) scale(0.144 -0.144)">+ <defs>+ <path id="DejaVuSans-2f" d="M 628 4666+L 1259 4666+L 1259 531+L 3531 531+L 3531 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-52" d="M 1959 3097+Q 1497 3097 1228 2736+Q 959 2375 959 1747+Q 959 1119 1226 758+Q 1494 397 1959 397+Q 2419 397 2687 759+Q 2956 1122 2956 1747+Q 2956 2369 2687 2733+Q 2419 3097 1959 3097+z+M 1959 3584+Q 2709 3584 3137 3096+Q 3566 2609 3566 1747+Q 3566 888 3137 398+Q 2709 -91 1959 -91+Q 1206 -91 779 398+Q 353 888 353 1747+Q 353 2609 779 3096+Q 1206 3584 1959 3584+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-51" d="M 3513 2113+L 3513 0+L 2938 0+L 2938 2094+Q 2938 2591 2744 2837+Q 2550 3084 2163 3084+Q 1697 3084 1428 2787+Q 1159 2491 1159 1978+L 1159 0+L 581 0+L 581 3500+L 1159 3500+L 1159 2956+Q 1366 3272 1645 3428+Q 1925 3584 2291 3584+Q 2894 3584 3203 3211+Q 3513 2838 3513 2113+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-4a" d="M 2906 1791+Q 2906 2416 2648 2759+Q 2391 3103 1925 3103+Q 1463 3103 1205 2759+Q 947 2416 947 1791+Q 947 1169 1205 825+Q 1463 481 1925 481+Q 2391 481 2648 825+Q 2906 1169 2906 1791+z+M 3481 434+Q 3481 -459 3084 -895+Q 2688 -1331 1869 -1331+Q 1566 -1331 1297 -1286+Q 1028 -1241 775 -1147+L 775 -588+Q 1028 -725 1275 -790+Q 1522 -856 1778 -856+Q 2344 -856 2625 -561+Q 2906 -266 2906 331+L 2906 616+Q 2728 306 2450 153+Q 2172 0 1784 0+Q 1141 0 747 490+Q 353 981 353 1791+Q 353 2603 747 3093+Q 1141 3584 1784 3584+Q 2172 3584 2450 3431+Q 2728 3278 2906 2969+L 2906 3500+L 3481 3500+L 3481 434+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-3" transform="scale(0.015625)"/>+ <path id="DejaVuSans-46" d="M 3122 3366+L 3122 2828+Q 2878 2963 2633 3030+Q 2388 3097 2138 3097+Q 1578 3097 1268 2742+Q 959 2388 959 1747+Q 959 1106 1268 751+Q 1578 397 2138 397+Q 2388 397 2633 464+Q 2878 531 3122 666+L 3122 134+Q 2881 22 2623 -34+Q 2366 -91 2075 -91+Q 1284 -91 818 406+Q 353 903 353 1747+Q 353 2603 823 3093+Q 1294 3584 2113 3584+Q 2378 3584 2631 3529+Q 2884 3475 3122 3366+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-57" d="M 1172 4494+L 1172 3500+L 2356 3500+L 2356 3053+L 1172 3053+L 1172 1153+Q 1172 725 1289 603+Q 1406 481 1766 481+L 2356 481+L 2356 0+L 1766 0+Q 1100 0 847 248+Q 594 497 594 1153+L 594 3053+L 172 3053+L 172 3500+L 594 3500+L 594 4494+L 1172 4494+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-48" d="M 3597 1894+L 3597 1613+L 953 1613+Q 991 1019 1311 708+Q 1631 397 2203 397+Q 2534 397 2845 478+Q 3156 559 3463 722+L 3463 178+Q 3153 47 2828 -22+Q 2503 -91 2169 -91+Q 1331 -91 842 396+Q 353 884 353 1716+Q 353 2575 817 3079+Q 1281 3584 2069 3584+Q 2775 3584 3186 3129+Q 3597 2675 3597 1894+z+M 3022 2063+Q 3016 2534 2758 2815+Q 2500 3097 2075 3097+Q 1594 3097 1305 2825+Q 1016 2553 972 2059+L 3022 2063+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-5b" d="M 3513 3500+L 2247 1797+L 3578 0+L 2900 0+L 1881 1375+L 863 0+L 184 0+L 1544 1831+L 300 3500+L 978 3500+L 1906 2253+L 2834 3500+L 3513 3500+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-2f"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(53.96875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(115.15625 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(178.53125 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(242.015625 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(273.796875 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(328.78125 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(389.96875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(453.34375 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(492.546875 0)"/>+ <use xlink:href="#DejaVuSans-5b" transform="translate(552.328125 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(611.515625 0)"/>+ </g>+ </g>+ <g id="text_2">+ <!-- Math -->+ <g style="fill: #222222" transform="translate(734.389875 35.6085) scale(0.144 -0.144)">+ <defs>+ <path id="DejaVuSans-30" d="M 628 4666+L 1569 4666+L 2759 1491+L 3956 4666+L 4897 4666+L 4897 0+L 4281 0+L 4281 4097+L 3078 897+L 2444 897+L 1241 4097+L 1241 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-44" d="M 2194 1759+Q 1497 1759 1228 1600+Q 959 1441 959 1056+Q 959 750 1161 570+Q 1363 391 1709 391+Q 2188 391 2477 730+Q 2766 1069 2766 1631+L 2766 1759+L 2194 1759+z+M 3341 1997+L 3341 0+L 2766 0+L 2766 531+Q 2569 213 2275 61+Q 1981 -91 1556 -91+Q 1019 -91 701 211+Q 384 513 384 1019+Q 384 1609 779 1909+Q 1175 2209 1959 2209+L 2766 2209+L 2766 2266+Q 2766 2663 2505 2880+Q 2244 3097 1772 3097+Q 1472 3097 1187 3025+Q 903 2953 641 2809+L 641 3341+Q 956 3463 1253 3523+Q 1550 3584 1831 3584+Q 2591 3584 2966 3190+Q 3341 2797 3341 1997+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-4b" d="M 3513 2113+L 3513 0+L 2938 0+L 2938 2094+Q 2938 2591 2744 2837+Q 2550 3084 2163 3084+Q 1697 3084 1428 2787+Q 1159 2491 1159 1978+L 1159 0+L 581 0+L 581 4863+L 1159 4863+L 1159 2956+Q 1366 3272 1645 3428+Q 1925 3584 2291 3584+Q 2894 3584 3203 3211+Q 3513 2838 3513 2113+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-30"/>+ <use xlink:href="#DejaVuSans-44" transform="translate(86.28125 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(147.5625 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(186.765625 0)"/>+ </g>+ </g>+ <g id="text_3">+ <!-- Code -->+ <g style="fill: #222222" transform="translate(867.88575 35.6085) scale(0.144 -0.144)">+ <defs>+ <path id="DejaVuSans-26" d="M 4122 4306+L 4122 3641+Q 3803 3938 3442 4084+Q 3081 4231 2675 4231+Q 1875 4231 1450 3742+Q 1025 3253 1025 2328+Q 1025 1406 1450 917+Q 1875 428 2675 428+Q 3081 428 3442 575+Q 3803 722 4122 1019+L 4122 359+Q 3791 134 3420 21+Q 3050 -91 2638 -91+Q 1578 -91 968 557+Q 359 1206 359 2328+Q 359 3453 968 4101+Q 1578 4750 2638 4750+Q 3056 4750 3426 4639+Q 3797 4528 4122 4306+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-47" d="M 2906 2969+L 2906 4863+L 3481 4863+L 3481 0+L 2906 0+L 2906 525+Q 2725 213 2448 61+Q 2172 -91 1784 -91+Q 1150 -91 751 415+Q 353 922 353 1747+Q 353 2572 751 3078+Q 1150 3584 1784 3584+Q 2172 3584 2448 3432+Q 2725 3281 2906 2969+z+M 947 1747+Q 947 1113 1208 752+Q 1469 391 1925 391+Q 2381 391 2643 752+Q 2906 1113 2906 1747+Q 2906 2381 2643 2742+Q 2381 3103 1925 3103+Q 1469 3103 1208 2742+Q 947 2381 947 1747+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-26"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(69.828125 0)"/>+ <use xlink:href="#DejaVuSans-47" transform="translate(131.015625 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(194.5 0)"/>+ </g>+ </g>+ <g id="text_4">+ <!-- Knowledge -->+ <g style="fill: #222222" transform="translate(936.019125 34.2135) scale(0.144 -0.144)">+ <defs>+ <path id="DejaVuSans-2e" d="M 628 4666+L 1259 4666+L 1259 2694+L 3353 4666+L 4166 4666+L 1850 2491+L 4331 0+L 3500 0+L 1259 2247+L 1259 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-5a" d="M 269 3500+L 844 3500+L 1563 769+L 2278 3500+L 2956 3500+L 3675 769+L 4391 3500+L 4966 3500+L 4050 0+L 3372 0+L 2619 2869+L 1863 0+L 1184 0+L 269 3500+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-4f" d="M 603 4863+L 1178 4863+L 1178 0+L 603 0+L 603 4863+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-2e"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(65.578125 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(128.953125 0)"/>+ <use xlink:href="#DejaVuSans-5a" transform="translate(190.140625 0)"/>+ <use xlink:href="#DejaVuSans-4f" transform="translate(271.921875 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(299.703125 0)"/>+ <use xlink:href="#DejaVuSans-47" transform="translate(361.234375 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(424.71875 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(488.203125 0)"/>+ </g>+ </g>+ <g id="text_5">+ <!-- Model -->+ <g style="fill: #222222" transform="translate(88.631887 62.516925) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-Bold-30" d="M 588 4666+L 2119 4666+L 3181 2169+L 4250 4666+L 5778 4666+L 5778 0+L 4641 0+L 4641 3413+L 3566 897+L 2803 897+L 1728 3413+L 1728 0+L 588 0+L 588 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-52" d="M 2203 2784+Q 1831 2784 1636 2517+Q 1441 2250 1441 1747+Q 1441 1244 1636 976+Q 1831 709 2203 709+Q 2569 709 2762 976+Q 2956 1244 2956 1747+Q 2956 2250 2762 2517+Q 2569 2784 2203 2784+z+M 2203 3584+Q 3106 3584 3614 3096+Q 4122 2609 4122 1747+Q 4122 884 3614 396+Q 3106 -91 2203 -91+Q 1297 -91 786 396+Q 275 884 275 1747+Q 275 2609 786 3096+Q 1297 3584 2203 3584+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-47" d="M 2919 2988+L 2919 4863+L 4044 4863+L 4044 0+L 2919 0+L 2919 506+Q 2688 197 2409 53+Q 2131 -91 1766 -91+Q 1119 -91 703 423+Q 288 938 288 1747+Q 288 2556 703 3070+Q 1119 3584 1766 3584+Q 2128 3584 2408 3439+Q 2688 3294 2919 2988+z+M 2181 722+Q 2541 722 2730 984+Q 2919 1247 2919 1747+Q 2919 2247 2730 2509+Q 2541 2772 2181 2772+Q 1825 2772 1636 2509+Q 1447 2247 1447 1747+Q 1447 1247 1636 984+Q 1825 722 2181 722+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-48" d="M 4031 1759+L 4031 1441+L 1416 1441+Q 1456 1047 1700 850+Q 1944 653 2381 653+Q 2734 653 3104 758+Q 3475 863 3866 1075+L 3866 213+Q 3469 63 3072 -14+Q 2675 -91 2278 -91+Q 1328 -91 801 392+Q 275 875 275 1747+Q 275 2603 792 3093+Q 1309 3584 2216 3584+Q 3041 3584 3536 3087+Q 4031 2591 4031 1759+z+M 2881 2131+Q 2881 2450 2695 2645+Q 2509 2841 2209 2841+Q 1884 2841 1681 2658+Q 1478 2475 1428 2131+L 2881 2131+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-4f" d="M 538 4863+L 1656 4863+L 1656 0+L 538 0+L 538 4863+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-Bold-30"/>+ <use xlink:href="#DejaVuSans-Bold-52" transform="translate(99.515625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-47" transform="translate(168.21875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-48" transform="translate(239.796875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-4f" transform="translate(307.625 0)"/>+ </g>+ </g>+ <g id="text_6">+ <!-- Method -->+ <g style="fill: #222222" transform="translate(198 62.516925) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-Bold-57" d="M 1759 4494+L 1759 3500+L 2913 3500+L 2913 2700+L 1759 2700+L 1759 1216+Q 1759 972 1856 886+Q 1953 800 2241 800+L 2816 800+L 2816 0+L 1856 0+Q 1194 0 917 276+Q 641 553 641 1216+L 641 2700+L 84 2700+L 84 3500+L 641 3500+L 641 4494+L 1759 4494+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-4b" d="M 4056 2131+L 4056 0+L 2931 0+L 2931 347+L 2931 1625+Q 2931 2084 2911 2256+Q 2891 2428 2841 2509+Q 2775 2619 2662 2680+Q 2550 2741 2406 2741+Q 2056 2741 1856 2470+Q 1656 2200 1656 1722+L 1656 0+L 538 0+L 538 4863+L 1656 4863+L 1656 2988+Q 1909 3294 2193 3439+Q 2478 3584 2822 3584+Q 3428 3584 3742 3212+Q 4056 2841 4056 2131+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-Bold-30"/>+ <use xlink:href="#DejaVuSans-Bold-48" transform="translate(99.515625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-57" transform="translate(167.34375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-4b" transform="translate(215.140625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-52" transform="translate(286.328125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-47" transform="translate(355.03125 0)"/>+ </g>+ </g>+ <g id="text_7">+ <!-- LB-Pro -->+ <g style="fill: #222222" transform="translate(411.7401 62.284219) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-Bold-2f" d="M 588 4666+L 1791 4666+L 1791 909+L 3903 909+L 3903 0+L 588 0+L 588 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-25" d="M 2456 2859+Q 2741 2859 2887 2984+Q 3034 3109 3034 3353+Q 3034 3594 2887 3720+Q 2741 3847 2456 3847+L 1791 3847+L 1791 2859+L 2456 2859+z+M 2497 819+Q 2859 819 3042 972+Q 3225 1125 3225 1434+Q 3225 1738 3044 1889+Q 2863 2041 2497 2041+L 1791 2041+L 1791 819+L 2497 819+z+M 3616 2497+Q 4003 2384 4215 2081+Q 4428 1778 4428 1338+Q 4428 663 3972 331+Q 3516 0 2584 0+L 588 0+L 588 4666+L 2394 4666+Q 3366 4666 3802 4372+Q 4238 4078 4238 3431+Q 4238 3091 4078 2852+Q 3919 2613 3616 2497+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-10" d="M 347 2297+L 2309 2297+L 2309 1388+L 347 1388+L 347 2297+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-33" d="M 588 4666+L 2584 4666+Q 3475 4666 3951 4270+Q 4428 3875 4428 3144+Q 4428 2409 3951 2014+Q 3475 1619 2584 1619+L 1791 1619+L 1791 0+L 588 0+L 588 4666+z+M 1791 3794+L 1791 2491+L 2456 2491+Q 2806 2491 2997 2661+Q 3188 2831 3188 3144+Q 3188 3456 2997 3625+Q 2806 3794 2456 3794+L 1791 3794+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-55" d="M 3138 2547+Q 2991 2616 2845 2648+Q 2700 2681 2553 2681+Q 2122 2681 1889 2404+Q 1656 2128 1656 1613+L 1656 0+L 538 0+L 538 3500+L 1656 3500+L 1656 2925+Q 1872 3269 2151 3426+Q 2431 3584 2822 3584+Q 2878 3584 2943 3579+Q 3009 3575 3134 3559+L 3138 2547+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-Bold-2f"/>+ <use xlink:href="#DejaVuSans-Bold-25" transform="translate(63.71875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-10" transform="translate(139.9375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-33" transform="translate(181.4375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-55" transform="translate(254.734375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-52" transform="translate(304.046875 0)"/>+ </g>+ </g>+ <g id="text_8">+ <!-- MRCR -->+ <g style="fill: #222222" transform="translate(504.485887 62.383444) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-Bold-35" d="M 2297 2597+Q 2675 2597 2839 2737+Q 3003 2878 3003 3200+Q 3003 3519 2839 3656+Q 2675 3794 2297 3794+L 1791 3794+L 1791 2597+L 2297 2597+z+M 1791 1766+L 1791 0+L 588 0+L 588 4666+L 2425 4666+Q 3347 4666 3776 4356+Q 4206 4047 4206 3378+Q 4206 2916 3982 2619+Q 3759 2322 3309 2181+Q 3556 2125 3751 1926+Q 3947 1728 4147 1325+L 4800 0+L 3519 0+L 2950 1159+Q 2778 1509 2601 1637+Q 2425 1766 2131 1766+L 1791 1766+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-26" d="M 4288 256+Q 3956 84 3597 -3+Q 3238 -91 2847 -91+Q 1681 -91 1000 561+Q 319 1213 319 2328+Q 319 3447 1000 4098+Q 1681 4750 2847 4750+Q 3238 4750 3597 4662+Q 3956 4575 4288 4403+L 4288 3438+Q 3953 3666 3628 3772+Q 3303 3878 2944 3878+Q 2300 3878 1931 3465+Q 1563 3053 1563 2328+Q 1563 1606 1931 1193+Q 2300 781 2944 781+Q 3303 781 3628 887+Q 3953 994 4288 1222+L 4288 256+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-Bold-30"/>+ <use xlink:href="#DejaVuSans-Bold-35" transform="translate(99.515625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-26" transform="translate(176.515625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-35" transform="translate(249.90625 0)"/>+ </g>+ </g>+ <g id="text_9">+ <!-- RULER -->+ <g style="fill: #222222" transform="translate(590.717081 62.284219) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-Bold-38" d="M 588 4666+L 1791 4666+L 1791 1869+Q 1791 1291 1980 1042+Q 2169 794 2597 794+Q 3028 794 3217 1042+Q 3406 1291 3406 1869+L 3406 4666+L 4609 4666+L 4609 1869+Q 4609 878 4112 393+Q 3616 -91 2597 -91+Q 1581 -91 1084 393+Q 588 878 588 1869+L 588 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-28" d="M 588 4666+L 3834 4666+L 3834 3756+L 1791 3756+L 1791 2888+L 3713 2888+L 3713 1978+L 1791 1978+L 1791 909+L 3903 909+L 3903 0+L 588 0+L 588 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-Bold-35"/>+ <use xlink:href="#DejaVuSans-Bold-38" transform="translate(77 0)"/>+ <use xlink:href="#DejaVuSans-Bold-2f" transform="translate(158.203125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-28" transform="translate(221.921875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-35" transform="translate(290.234375 0)"/>+ </g>+ </g>+ <g id="text_10">+ <!-- AIME24 -->+ <g style="fill: #222222" transform="translate(675.888694 62.490937) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-Bold-24" d="M 3419 850+L 1538 850+L 1241 0+L 31 0+L 1759 4666+L 3194 4666+L 4922 0+L 3713 0+L 3419 850+z+M 1838 1716+L 3116 1716+L 2478 3572+L 1838 1716+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-2c" d="M 588 4666+L 1791 4666+L 1791 0+L 588 0+L 588 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-15" d="M 1844 884+L 3897 884+L 3897 0+L 506 0+L 506 884+L 2209 2388+Q 2438 2594 2547 2791+Q 2656 2988 2656 3200+Q 2656 3528 2436 3728+Q 2216 3928 1850 3928+Q 1569 3928 1234 3808+Q 900 3688 519 3450+L 519 4475+Q 925 4609 1322 4679+Q 1719 4750 2100 4750+Q 2938 4750 3402 4381+Q 3866 4013 3866 3353+Q 3866 2972 3669 2642+Q 3472 2313 2841 1759+L 1844 884+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-17" d="M 2356 3675+L 1038 1722+L 2356 1722+L 2356 3675+z+M 2156 4666+L 3494 4666+L 3494 1722+L 4159 1722+L 4159 850+L 3494 850+L 3494 0+L 2356 0+L 2356 850+L 288 850+L 288 1881+L 2156 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-Bold-24"/>+ <use xlink:href="#DejaVuSans-Bold-2c" transform="translate(77.390625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-30" transform="translate(114.59375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-28" transform="translate(214.109375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-15" transform="translate(282.421875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-17" transform="translate(352 0)"/>+ </g>+ </g>+ <g id="text_11">+ <!-- AIME25 -->+ <g style="fill: #222222" transform="translate(765.168694 62.383444) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-Bold-18" d="M 678 4666+L 3669 4666+L 3669 3781+L 1638 3781+L 1638 3059+Q 1775 3097 1914 3117+Q 2053 3138 2203 3138+Q 3056 3138 3531 2711+Q 4006 2284 4006 1522+Q 4006 766 3489 337+Q 2972 -91 2053 -91+Q 1656 -91 1267 -14+Q 878 63 494 219+L 494 1166+Q 875 947 1217 837+Q 1559 728 1863 728+Q 2300 728 2551 942+Q 2803 1156 2803 1522+Q 2803 1891 2551 2103+Q 2300 2316 1863 2316+Q 1603 2316 1309 2248+Q 1016 2181 678 2041+L 678 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-Bold-24"/>+ <use xlink:href="#DejaVuSans-Bold-2c" transform="translate(77.390625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-30" transform="translate(114.59375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-28" transform="translate(214.109375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-15" transform="translate(282.421875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-18" transform="translate(352 0)"/>+ </g>+ </g>+ <g id="text_12">+ <!-- LCB v6 -->+ <g style="fill: #222222" transform="translate(857.372287 62.383444) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-Bold-3" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-59" d="M 97 3500+L 1216 3500+L 2088 1081+L 2956 3500+L 4078 3500+L 2700 0+L 1472 0+L 97 3500+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-19" d="M 2316 2303+Q 2000 2303 1842 2098+Q 1684 1894 1684 1484+Q 1684 1075 1842 870+Q 2000 666 2316 666+Q 2634 666 2792 870+Q 2950 1075 2950 1484+Q 2950 1894 2792 2098+Q 2634 2303 2316 2303+z+M 3803 4544+L 3803 3681+Q 3506 3822 3243 3889+Q 2981 3956 2731 3956+Q 2194 3956 1894 3657+Q 1594 3359 1544 2772+Q 1750 2925 1990 3001+Q 2231 3078 2516 3078+Q 3231 3078 3670 2659+Q 4109 2241 4109 1563+Q 4109 813 3618 361+Q 3128 -91 2303 -91+Q 1394 -91 895 523+Q 397 1138 397 2266+Q 397 3422 980 4083+Q 1563 4744 2578 4744+Q 2900 4744 3203 4694+Q 3506 4644 3803 4544+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-Bold-2f"/>+ <use xlink:href="#DejaVuSans-Bold-26" transform="translate(63.71875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-25" transform="translate(137.109375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-3" transform="translate(213.328125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-59" transform="translate(248.140625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-19" transform="translate(313.328125 0)"/>+ </g>+ </g>+ <g id="text_13">+ <!-- GPQA -->+ <g style="fill: #222222" transform="translate(951.575737 61.38765) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-Bold-2a" d="M 4781 347+Q 4331 128 3847 18+Q 3363 -91 2847 -91+Q 1681 -91 1000 561+Q 319 1213 319 2328+Q 319 3456 1012 4103+Q 1706 4750 2913 4750+Q 3378 4750 3804 4662+Q 4231 4575 4609 4403+L 4609 3438+Q 4219 3659 3833 3768+Q 3447 3878 3059 3878+Q 2341 3878 1952 3476+Q 1563 3075 1563 2328+Q 1563 1588 1938 1184+Q 2313 781 3003 781+Q 3191 781 3352 804+Q 3513 828 3641 878+L 3641 1784+L 2906 1784+L 2906 2591+L 4781 2591+L 4781 347+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-34" d="M 2847 -84+L 2753 -84+Q 1600 -84 959 553+Q 319 1191 319 2328+Q 319 3463 958 4106+Q 1597 4750 2719 4750+Q 3853 4750 4486 4112+Q 5119 3475 5119 2328+Q 5119 1541 4783 972+Q 4447 403 3816 116+L 4756 -934+L 3609 -934+L 2847 -84+z+M 2719 3878+Q 2169 3878 1866 3472+Q 1563 3066 1563 2328+Q 1563 1578 1859 1179+Q 2156 781 2719 781+Q 3272 781 3575 1187+Q 3878 1594 3878 2328+Q 3878 3066 3575 3472+Q 3272 3878 2719 3878+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-Bold-2a"/>+ <use xlink:href="#DejaVuSans-Bold-33" transform="translate(82.078125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-34" transform="translate(155.375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-24" transform="translate(240.390625 0)"/>+ </g>+ </g>+ <g id="text_14">+ <!-- Avg. -->+ <g style="fill: #222222" transform="translate(1047.527494 60.760406) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-Bold-4a" d="M 2919 594+Q 2688 288 2409 144+Q 2131 0 1766 0+Q 1125 0 706 504+Q 288 1009 288 1791+Q 288 2575 706 3076+Q 1125 3578 1766 3578+Q 2131 3578 2409 3434+Q 2688 3291 2919 2981+L 2919 3500+L 4044 3500+L 4044 353+Q 4044 -491 3511 -936+Q 2978 -1381 1966 -1381+Q 1638 -1381 1331 -1331+Q 1025 -1281 716 -1178+L 716 -306+Q 1009 -475 1290 -558+Q 1572 -641 1856 -641+Q 2406 -641 2662 -400+Q 2919 -159 2919 353+L 2919 594+z+M 2181 2772+Q 1834 2772 1640 2515+Q 1447 2259 1447 1791+Q 1447 1309 1634 1061+Q 1822 813 2181 813+Q 2531 813 2725 1069+Q 2919 1325 2919 1791+Q 2919 2259 2725 2515+Q 2531 2772 2181 2772+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-11" d="M 653 1209+L 1778 1209+L 1778 0+L 653 0+L 653 1209+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-Bold-24"/>+ <use xlink:href="#DejaVuSans-Bold-59" transform="translate(73.828125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-4a" transform="translate(139.015625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-11" transform="translate(210.59375 0)"/>+ </g>+ </g>+ <g id="text_15">+ <!-- BF16 KV -->+ <g style="fill: #222222" transform="translate(198 89.383444) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-25" d="M 1259 2228+L 1259 519+L 2272 519+Q 2781 519 3026 730+Q 3272 941 3272 1375+Q 3272 1813 3026 2020+Q 2781 2228 2272 2228+L 1259 2228+z+M 1259 4147+L 1259 2741+L 2194 2741+Q 2656 2741 2882 2914+Q 3109 3088 3109 3444+Q 3109 3797 2882 3972+Q 2656 4147 2194 4147+L 1259 4147+z+M 628 4666+L 2241 4666+Q 2963 4666 3353 4366+Q 3744 4066 3744 3513+Q 3744 3084 3544 2831+Q 3344 2578 2956 2516+Q 3422 2416 3680 2098+Q 3938 1781 3938 1306+Q 3938 681 3513 340+Q 3088 0 2303 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-29" d="M 628 4666+L 3309 4666+L 3309 4134+L 1259 4134+L 1259 2759+L 3109 2759+L 3109 2228+L 1259 2228+L 1259 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-14" d="M 794 531+L 1825 531+L 1825 4091+L 703 3866+L 703 4441+L 1819 4666+L 2450 4666+L 2450 531+L 3481 531+L 3481 0+L 794 0+L 794 531+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-19" d="M 2113 2584+Q 1688 2584 1439 2293+Q 1191 2003 1191 1497+Q 1191 994 1439 701+Q 1688 409 2113 409+Q 2538 409 2786 701+Q 3034 994 3034 1497+Q 3034 2003 2786 2293+Q 2538 2584 2113 2584+z+M 3366 4563+L 3366 3988+Q 3128 4100 2886 4159+Q 2644 4219 2406 4219+Q 1781 4219 1451 3797+Q 1122 3375 1075 2522+Q 1259 2794 1537 2939+Q 1816 3084 2150 3084+Q 2853 3084 3261 2657+Q 3669 2231 3669 1497+Q 3669 778 3244 343+Q 2819 -91 2113 -91+Q 1303 -91 875 529+Q 447 1150 447 2328+Q 447 3434 972 4092+Q 1497 4750 2381 4750+Q 2619 4750 2861 4703+Q 3103 4656 3366 4563+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-39" d="M 1831 0+L 50 4666+L 709 4666+L 2188 738+L 3669 4666+L 4325 4666+L 2547 0+L 1831 0+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-25"/>+ <use xlink:href="#DejaVuSans-29" transform="translate(68.609375 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(126.125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(189.75 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(253.375 0)"/>+ <use xlink:href="#DejaVuSans-2e" transform="translate(285.15625 0)"/>+ <use xlink:href="#DejaVuSans-39" transform="translate(350.734375 0)"/>+ </g>+ </g>+ <g id="text_16">+ <!-- 53.27 -->+ <g style="fill: #222222" transform="translate(417.246525 89.645512) scale(0.1584 -0.1584)">+ <defs>+ <path id="DejaVuSans-18" d="M 691 4666+L 3169 4666+L 3169 4134+L 1269 4134+L 1269 2991+Q 1406 3038 1543 3061+Q 1681 3084 1819 3084+Q 2600 3084 3056 2656+Q 3513 2228 3513 1497+Q 3513 744 3044 326+Q 2575 -91 1722 -91+Q 1428 -91 1123 -41+Q 819 9 494 109+L 494 744+Q 775 591 1075 516+Q 1375 441 1709 441+Q 2250 441 2565 725+Q 2881 1009 2881 1497+Q 2881 1984 2565 2268+Q 2250 2553 1709 2553+Q 1456 2553 1204 2497+Q 953 2441 691 2322+L 691 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-16" d="M 2597 2516+Q 3050 2419 3304 2112+Q 3559 1806 3559 1356+Q 3559 666 3084 287+Q 2609 -91 1734 -91+Q 1441 -91 1130 -33+Q 819 25 488 141+L 488 750+Q 750 597 1062 519+Q 1375 441 1716 441+Q 2309 441 2620 675+Q 2931 909 2931 1356+Q 2931 1769 2642 2001+Q 2353 2234 1838 2234+L 1294 2234+L 1294 2753+L 1863 2753+Q 2328 2753 2575 2939+Q 2822 3125 2822 3475+Q 2822 3834 2567 4026+Q 2313 4219 1838 4219+Q 1578 4219 1281 4162+Q 984 4106 628 3988+L 628 4550+Q 988 4650 1302 4700+Q 1616 4750 1894 4750+Q 2613 4750 3031 4423+Q 3450 4097 3450 3541+Q 3450 3153 3228 2886+Q 3006 2619 2597 2516+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-11" d="M 684 794+L 1344 794+L 1344 0+L 684 0+L 684 794+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-15" d="M 1228 531+L 3431 531+L 3431 0+L 469 0+L 469 531+Q 828 903 1448 1529+Q 2069 2156 2228 2338+Q 2531 2678 2651 2914+Q 2772 3150 2772 3378+Q 2772 3750 2511 3984+Q 2250 4219 1831 4219+Q 1534 4219 1204 4116+Q 875 4013 500 3803+L 500 4441+Q 881 4594 1212 4672+Q 1544 4750 1819 4750+Q 2544 4750 2975 4387+Q 3406 4025 3406 3419+Q 3406 3131 3298 2873+Q 3191 2616 2906 2266+Q 2828 2175 2409 1742+Q 1991 1309 1228 531+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-1a" d="M 525 4666+L 3525 4666+L 3525 4397+L 1831 0+L 1172 0+L 2766 4134+L 525 4134+L 525 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_17">+ <!-- 38.82 -->+ <g style="fill: #222222" transform="translate(506.526525 89.645512) scale(0.1584 -0.1584)">+ <defs>+ <path id="DejaVuSans-1b" d="M 2034 2216+Q 1584 2216 1326 1975+Q 1069 1734 1069 1313+Q 1069 891 1326 650+Q 1584 409 2034 409+Q 2484 409 2743 651+Q 3003 894 3003 1313+Q 3003 1734 2745 1975+Q 2488 2216 2034 2216+z+M 1403 2484+Q 997 2584 770 2862+Q 544 3141 544 3541+Q 544 4100 942 4425+Q 1341 4750 2034 4750+Q 2731 4750 3128 4425+Q 3525 4100 3525 3541+Q 3525 3141 3298 2862+Q 3072 2584 2669 2484+Q 3125 2378 3379 2068+Q 3634 1759 3634 1313+Q 3634 634 3220 271+Q 2806 -91 2034 -91+Q 1263 -91 848 271+Q 434 634 434 1313+Q 434 1759 690 2068+Q 947 2378 1403 2484+z+M 1172 3481+Q 1172 3119 1398 2916+Q 1625 2713 2034 2713+Q 2441 2713 2670 2916+Q 2900 3119 2900 3481+Q 2900 3844 2670 4047+Q 2441 4250 2034 4250+Q 1625 4250 1398 4047+Q 1172 3844 1172 3481+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_18">+ <!-- 96.28 -->+ <g style="fill: #222222" transform="translate(595.806525 89.645512) scale(0.1584 -0.1584)">+ <defs>+ <path id="DejaVuSans-1c" d="M 703 97+L 703 672+Q 941 559 1184 500+Q 1428 441 1663 441+Q 2288 441 2617 861+Q 2947 1281 2994 2138+Q 2813 1869 2534 1725+Q 2256 1581 1919 1581+Q 1219 1581 811 2004+Q 403 2428 403 3163+Q 403 3881 828 4315+Q 1253 4750 1959 4750+Q 2769 4750 3195 4129+Q 3622 3509 3622 2328+Q 3622 1225 3098 567+Q 2575 -91 1691 -91+Q 1453 -91 1209 -44+Q 966 3 703 97+z+M 1959 2075+Q 2384 2075 2632 2365+Q 2881 2656 2881 3163+Q 2881 3666 2632 3958+Q 2384 4250 1959 4250+Q 1534 4250 1286 3958+Q 1038 3666 1038 3163+Q 1038 2656 1286 2365+Q 1534 2075 1959 2075+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_19">+ <!-- 83.44 -->+ <g style="fill: #222222" transform="translate(685.086525 89.645512) scale(0.1584 -0.1584)">+ <defs>+ <path id="DejaVuSans-17" d="M 2419 4116+L 825 1625+L 2419 1625+L 2419 4116+z+M 2253 4666+L 3047 4666+L 3047 1625+L 3713 1625+L 3713 1100+L 3047 1100+L 3047 0+L 2419 0+L 2419 1100+L 313 1100+L 313 1709+L 2253 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_20">+ <!-- 81.15 -->+ <g style="fill: #222222" transform="translate(774.366525 89.645512) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_21">+ <!-- 53.86 -->+ <g style="fill: #222222" transform="translate(863.646525 89.645512) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_22">+ <!-- 66.98 -->+ <g style="fill: #222222" transform="translate(952.926525 89.645512) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_23">+ <!-- 67.69 -->+ <g style="fill: #222222" transform="translate(1043.646525 89.645512) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_24">+ <!-- FP8 KV -->+ <g style="fill: #222222" transform="translate(198 111.703444) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-33" d="M 1259 4147+L 1259 2394+L 2053 2394+Q 2494 2394 2734 2622+Q 2975 2850 2975 3272+Q 2975 3691 2734 3919+Q 2494 4147 2053 4147+L 1259 4147+z+M 628 4666+L 2053 4666+Q 2838 4666 3239 4311+Q 3641 3956 3641 3272+Q 3641 2581 3239 2228+Q 2838 1875 2053 1875+L 1259 1875+L 1259 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-29"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(57.515625 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(117.8125 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(181.4375 0)"/>+ <use xlink:href="#DejaVuSans-2e" transform="translate(213.21875 0)"/>+ <use xlink:href="#DejaVuSans-39" transform="translate(278.796875 0)"/>+ </g>+ </g>+ <g id="text_25">+ <!-- 52.60 -->+ <g style="fill: #222222" transform="translate(417.246525 111.965512) scale(0.1584 -0.1584)">+ <defs>+ <path id="DejaVuSans-13" d="M 2034 4250+Q 1547 4250 1301 3770+Q 1056 3291 1056 2328+Q 1056 1369 1301 889+Q 1547 409 2034 409+Q 2525 409 2770 889+Q 3016 1369 3016 2328+Q 3016 3291 2770 3770+Q 2525 4250 2034 4250+z+M 2034 4750+Q 2819 4750 3233 4129+Q 3647 3509 3647 2328+Q 3647 1150 3233 529+Q 2819 -91 2034 -91+Q 1250 -91 836 529+Q 422 1150 422 2328+Q 422 3509 836 4129+Q 1250 4750 2034 4750+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_26">+ <!-- 37.67 -->+ <g style="fill: #222222" transform="translate(506.526525 111.965512) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_27">+ <!-- 94.42 -->+ <g style="fill: #222222" transform="translate(595.806525 111.965512) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_28">+ <!-- 82.29 -->+ <g style="fill: #222222" transform="translate(685.086525 111.965512) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_29">+ <!-- 78.75 -->+ <g style="fill: #222222" transform="translate(774.366525 111.965512) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_30">+ <!-- 50.86 -->+ <g style="fill: #222222" transform="translate(863.646525 111.965512) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_31">+ <!-- 66.86 -->+ <g style="fill: #222222" transform="translate(952.926525 111.965512) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_32">+ <!-- 66.21 -->+ <g style="fill: #222222" transform="translate(1043.646525 111.965512) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_33">+ <!-- TurboQuant-4bit-nc -->+ <g style="fill: #222222" transform="translate(198 133.289888) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-37" d="M -19 4666+L 3928 4666+L 3928 4134+L 2272 4134+L 2272 0+L 1638 0+L 1638 4134+L -19 4134+L -19 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-58" d="M 544 1381+L 544 3500+L 1119 3500+L 1119 1403+Q 1119 906 1312 657+Q 1506 409 1894 409+Q 2359 409 2629 706+Q 2900 1003 2900 1516+L 2900 3500+L 3475 3500+L 3475 0+L 2900 0+L 2900 538+Q 2691 219 2414 64+Q 2138 -91 1772 -91+Q 1169 -91 856 284+Q 544 659 544 1381+z+M 1991 3584+L 1991 3584+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-55" d="M 2631 2963+Q 2534 3019 2420 3045+Q 2306 3072 2169 3072+Q 1681 3072 1420 2755+Q 1159 2438 1159 1844+L 1159 0+L 581 0+L 581 3500+L 1159 3500+L 1159 2956+Q 1341 3275 1631 3429+Q 1922 3584 2338 3584+Q 2397 3584 2469 3576+Q 2541 3569 2628 3553+L 2631 2963+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-45" d="M 3116 1747+Q 3116 2381 2855 2742+Q 2594 3103 2138 3103+Q 1681 3103 1420 2742+Q 1159 2381 1159 1747+Q 1159 1113 1420 752+Q 1681 391 2138 391+Q 2594 391 2855 752+Q 3116 1113 3116 1747+z+M 1159 2969+Q 1341 3281 1617 3432+Q 1894 3584 2278 3584+Q 2916 3584 3314 3078+Q 3713 2572 3713 1747+Q 3713 922 3314 415+Q 2916 -91 2278 -91+Q 1894 -91 1617 61+Q 1341 213 1159 525+L 1159 0+L 581 0+L 581 4863+L 1159 4863+L 1159 2969+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-34" d="M 2522 4238+Q 1834 4238 1429 3725+Q 1025 3213 1025 2328+Q 1025 1447 1429 934+Q 1834 422 2522 422+Q 3209 422 3611 934+Q 4013 1447 4013 2328+Q 4013 3213 3611 3725+Q 3209 4238 2522 4238+z+M 3406 84+L 4238 -825+L 3475 -825+L 2784 -78+Q 2681 -84 2626 -87+Q 2572 -91 2522 -91+Q 1538 -91 948 567+Q 359 1225 359 2328+Q 359 3434 948 4092+Q 1538 4750 2522 4750+Q 3503 4750 4090 4092+Q 4678 3434 4678 2328+Q 4678 1516 4351 937+Q 4025 359 3406 84+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-10" d="M 313 2009+L 1997 2009+L 1997 1497+L 313 1497+L 313 2009+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-4c" d="M 603 3500+L 1178 3500+L 1178 0+L 603 0+L 603 3500+z+M 603 4863+L 1178 4863+L 1178 4134+L 603 4134+L 603 4863+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-37"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(45.890625 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(109.265625 0)"/>+ <use xlink:href="#DejaVuSans-45" transform="translate(150.375 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(213.859375 0)"/>+ <use xlink:href="#DejaVuSans-34" transform="translate(275.046875 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(353.765625 0)"/>+ <use xlink:href="#DejaVuSans-44" transform="translate(417.140625 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(478.421875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(541.796875 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(581 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(617.078125 0)"/>+ <use xlink:href="#DejaVuSans-45" transform="translate(680.703125 0)"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(744.1875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(771.96875 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(811.171875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(847.25 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(910.625 0)"/>+ </g>+ </g>+ <g id="text_34">+ <!-- 49.78 -->+ <g style="fill: #222222" transform="translate(417.246525 134.285513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-17"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_35">+ <!-- 32.91 -->+ <g style="fill: #222222" transform="translate(506.526525 134.285513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_36">+ <!-- 93.15 -->+ <g style="fill: #222222" transform="translate(595.806525 134.285513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_37">+ <!-- 77.40 -->+ <g style="fill: #222222" transform="translate(685.086525 134.285513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_38">+ <!-- 72.29 -->+ <g style="fill: #222222" transform="translate(774.366525 134.285513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_39">+ <!-- 47.43 -->+ <g style="fill: #222222" transform="translate(863.646525 134.285513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-17"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_40">+ <!-- 64.96 -->+ <g style="fill: #222222" transform="translate(952.926525 134.285513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_41">+ <!-- 62.56 -->+ <g style="fill: #222222" transform="translate(1043.646525 134.285513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_42">+ <!-- SAW-INT4 -->+ <g style="fill: #222222" transform="translate(198 156.343444) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-36" d="M 3425 4513+L 3425 3897+Q 3066 4069 2747 4153+Q 2428 4238 2131 4238+Q 1616 4238 1336 4038+Q 1056 3838 1056 3469+Q 1056 3159 1242 3001+Q 1428 2844 1947 2747+L 2328 2669+Q 3034 2534 3370 2195+Q 3706 1856 3706 1288+Q 3706 609 3251 259+Q 2797 -91 1919 -91+Q 1588 -91 1214 -16+Q 841 59 441 206+L 441 856+Q 825 641 1194 531+Q 1563 422 1919 422+Q 2459 422 2753 634+Q 3047 847 3047 1241+Q 3047 1584 2836 1778+Q 2625 1972 2144 2069+L 1759 2144+Q 1053 2284 737 2584+Q 422 2884 422 3419+Q 422 4038 858 4394+Q 1294 4750 2059 4750+Q 2388 4750 2728 4690+Q 3069 4631 3425 4513+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-24" d="M 2188 4044+L 1331 1722+L 3047 1722+L 2188 4044+z+M 1831 4666+L 2547 4666+L 4325 0+L 3669 0+L 3244 1197+L 1141 1197+L 716 0+L 50 0+L 1831 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-3a" d="M 213 4666+L 850 4666+L 1831 722+L 2809 4666+L 3519 4666+L 4500 722+L 5478 4666+L 6119 4666+L 4947 0+L 4153 0+L 3169 4050+L 2175 0+L 1381 0+L 213 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-2c" d="M 628 4666+L 1259 4666+L 1259 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-31" d="M 628 4666+L 1478 4666+L 3547 763+L 3547 4666+L 4159 4666+L 4159 0+L 3309 0+L 1241 3903+L 1241 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-36"/>+ <use xlink:href="#DejaVuSans-24" transform="translate(65.34375 0)"/>+ <use xlink:href="#DejaVuSans-3a" transform="translate(128.28125 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(223.109375 0)"/>+ <use xlink:href="#DejaVuSans-2c" transform="translate(259.1875 0)"/>+ <use xlink:href="#DejaVuSans-31" transform="translate(288.6875 0)"/>+ <use xlink:href="#DejaVuSans-37" transform="translate(363.5 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(424.578125 0)"/>+ </g>+ </g>+ <g id="text_43">+ <!-- 49.92 -->+ <g style="fill: #222222" transform="translate(417.246525 156.605513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-17"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_44">+ <!-- 31.57 -->+ <g style="fill: #222222" transform="translate(506.526525 156.605513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_45">+ <!-- 91.19 -->+ <g style="fill: #222222" transform="translate(595.806525 156.605513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_46">+ <!-- 79.69 -->+ <g style="fill: #222222" transform="translate(685.086525 156.605513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_47">+ <!-- 75.63 -->+ <g style="fill: #222222" transform="translate(774.366525 156.605513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_48">+ <!-- 52.86 -->+ <g style="fill: #222222" transform="translate(863.646525 156.605513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_49">+ <!-- 64.02 -->+ <g style="fill: #222222" transform="translate(952.926525 156.605513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_50">+ <!-- 63.55 -->+ <g style="fill: #222222" transform="translate(1043.646525 156.605513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_51">+ <!-- TRT-LLM-NVFP4 -->+ <g style="fill: #222222" transform="translate(198 178.671713) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-35" d="M 2841 2188+Q 3044 2119 3236 1894+Q 3428 1669 3622 1275+L 4263 0+L 3584 0+L 2988 1197+Q 2756 1666 2539 1819+Q 2322 1972 1947 1972+L 1259 1972+L 1259 0+L 628 0+L 628 4666+L 2053 4666+Q 2853 4666 3247 4331+Q 3641 3997 3641 3322+Q 3641 2881 3436 2590+Q 3231 2300 2841 2188+z+M 1259 4147+L 1259 2491+L 2053 2491+Q 2509 2491 2742 2702+Q 2975 2913 2975 3322+Q 2975 3731 2742 3939+Q 2509 4147 2053 4147+L 1259 4147+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-37"/>+ <use xlink:href="#DejaVuSans-35" transform="translate(61.078125 0)"/>+ <use xlink:href="#DejaVuSans-37" transform="translate(123.28125 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(175.1875 0)"/>+ <use xlink:href="#DejaVuSans-2f" transform="translate(211.265625 0)"/>+ <use xlink:href="#DejaVuSans-2f" transform="translate(266.984375 0)"/>+ <use xlink:href="#DejaVuSans-30" transform="translate(322.703125 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(408.984375 0)"/>+ <use xlink:href="#DejaVuSans-31" transform="translate(445.0625 0)"/>+ <use xlink:href="#DejaVuSans-39" transform="translate(519.875 0)"/>+ <use xlink:href="#DejaVuSans-29" transform="translate(588.28125 0)"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(645.796875 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(706.09375 0)"/>+ </g>+ </g>+ <g id="text_52">+ <!-- 44.89 -->+ <g style="fill: #222222" transform="translate(417.246525 178.925513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-17"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_53">+ <!-- 28.84 -->+ <g style="fill: #222222" transform="translate(506.526525 178.925513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-15"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_54">+ <!-- 80.04 -->+ <g style="fill: #222222" transform="translate(595.806525 178.925513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_55">+ <!-- 76.56 -->+ <g style="fill: #222222" transform="translate(685.086525 178.925513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_56">+ <!-- 72.29 -->+ <g style="fill: #222222" transform="translate(774.366525 178.925513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_57">+ <!-- 48.71 -->+ <g style="fill: #222222" transform="translate(863.646525 178.925513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-17"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_58">+ <!-- 63.01 -->+ <g style="fill: #222222" transform="translate(952.926525 178.925513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_59">+ <!-- 59.19 -->+ <g style="fill: #222222" transform="translate(1043.646525 178.925513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_60">+ <!-- BitDecoding-MXFP4 -->+ <g style="fill: #222222" transform="translate(198 199.652175) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-27" d="M 1259 4147+L 1259 519+L 2022 519+Q 2988 519 3436 956+Q 3884 1394 3884 2338+Q 3884 3275 3436 3711+Q 2988 4147 2022 4147+L 1259 4147+z+M 628 4666+L 1925 4666+Q 3281 4666 3915 4102+Q 4550 3538 4550 2338+Q 4550 1131 3912 565+Q 3275 0 1925 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-3b" d="M 403 4666+L 1081 4666+L 2241 2931+L 3406 4666+L 4084 4666+L 2584 2425+L 4184 0+L 3506 0+L 2194 1984+L 872 0+L 191 0+L 1856 2491+L 403 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-25"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(68.609375 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(96.390625 0)"/>+ <use xlink:href="#DejaVuSans-27" transform="translate(135.59375 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(212.59375 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(274.125 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(329.109375 0)"/>+ <use xlink:href="#DejaVuSans-47" transform="translate(390.296875 0)"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(453.78125 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(481.5625 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(544.9375 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(608.421875 0)"/>+ <use xlink:href="#DejaVuSans-30" transform="translate(644.5 0)"/>+ <use xlink:href="#DejaVuSans-3b" transform="translate(730.78125 0)"/>+ <use xlink:href="#DejaVuSans-29" transform="translate(799.28125 0)"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(856.796875 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(917.09375 0)"/>+ </g>+ </g>+ <g id="text_61">+ <!-- 52.96 -->+ <g style="fill: #222222" transform="translate(417.246525 201.245513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_62">+ <!-- 35.61 -->+ <g style="fill: #222222" transform="translate(506.526525 201.245513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_63">+ <!-- 94.46 -->+ <g style="fill: #222222" transform="translate(595.806525 201.245513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_64">+ <!-- 81.56 -->+ <g style="fill: #222222" transform="translate(685.086525 201.245513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_65">+ <!-- 80.42 -->+ <g style="fill: #222222" transform="translate(774.366525 201.245513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_66">+ <!-- 55.14 -->+ <g style="fill: #222222" transform="translate(863.646525 201.141563) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_67">+ <!-- 66.73 -->+ <g style="fill: #222222" transform="translate(952.926525 201.245513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_68">+ <!-- 66.70 -->+ <g style="fill: #222222" transform="translate(1043.646525 201.245513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_69">+ <!-- BitDecoding-NVFP4 -->+ <g style="fill: #222222" transform="translate(198 221.913113) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-Bold-4c" d="M 538 3500+L 1656 3500+L 1656 0+L 538 0+L 538 3500+z+M 538 4863+L 1656 4863+L 1656 3950+L 538 3950+L 538 4863+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-27" d="M 1791 3756+L 1791 909+L 2222 909+Q 2959 909 3348 1275+Q 3738 1641 3738 2338+Q 3738 3031 3350 3393+Q 2963 3756 2222 3756+L 1791 3756+z+M 588 4666+L 1856 4666+Q 2919 4666 3439 4514+Q 3959 4363 4331 4000+Q 4659 3684 4818 3271+Q 4978 2859 4978 2338+Q 4978 1809 4818 1395+Q 4659 981 4331 666+Q 3956 303 3431 151+Q 2906 0 1856 0+L 588 0+L 588 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-46" d="M 3366 3391+L 3366 2478+Q 3138 2634 2908 2709+Q 2678 2784 2431 2784+Q 1963 2784 1702 2511+Q 1441 2238 1441 1747+Q 1441 1256 1702 982+Q 1963 709 2431 709+Q 2694 709 2930 787+Q 3166 866 3366 1019+L 3366 103+Q 3103 6 2833 -42+Q 2563 -91 2291 -91+Q 1344 -91 809 395+Q 275 881 275 1747+Q 275 2613 809 3098+Q 1344 3584 2291 3584+Q 2566 3584 2833 3536+Q 3100 3488 3366 3391+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-51" d="M 4056 2131+L 4056 0+L 2931 0+L 2931 347+L 2931 1631+Q 2931 2084 2911 2256+Q 2891 2428 2841 2509+Q 2775 2619 2662 2680+Q 2550 2741 2406 2741+Q 2056 2741 1856 2470+Q 1656 2200 1656 1722+L 1656 0+L 538 0+L 538 3500+L 1656 3500+L 1656 2988+Q 1909 3294 2193 3439+Q 2478 3584 2822 3584+Q 3428 3584 3742 3212+Q 4056 2841 4056 2131+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-31" d="M 588 4666+L 1931 4666+L 3628 1466+L 3628 4666+L 4769 4666+L 4769 0+L 3425 0+L 1728 3200+L 1728 0+L 588 0+L 588 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-39" d="M 31 4666+L 1241 4666+L 2478 1222+L 3713 4666+L 4922 4666+L 3194 0+L 1759 0+L 31 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-29" d="M 588 4666+L 3834 4666+L 3834 3756+L 1791 3756+L 1791 2888+L 3713 2888+L 3713 1978+L 1791 1978+L 1791 0+L 588 0+L 588 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-Bold-25"/>+ <use xlink:href="#DejaVuSans-Bold-4c" transform="translate(76.21875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-57" transform="translate(110.5 0)"/>+ <use xlink:href="#DejaVuSans-Bold-27" transform="translate(158.296875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-48" transform="translate(241.3125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-46" transform="translate(309.140625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-52" transform="translate(368.421875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-47" transform="translate(437.125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-4c" transform="translate(508.703125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-51" transform="translate(542.984375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-4a" transform="translate(614.171875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-10" transform="translate(685.75 0)"/>+ <use xlink:href="#DejaVuSans-Bold-31" transform="translate(727.25 0)"/>+ <use xlink:href="#DejaVuSans-Bold-39" transform="translate(810.9375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-29" transform="translate(888.328125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-33" transform="translate(956.640625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-17" transform="translate(1029.9375 0)"/>+ </g>+ </g>+ <g id="text_70">+ <!-- 52.34 -->+ <g style="fill: #222222" transform="translate(417.246525 223.565513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_71">+ <!-- 36.68 -->+ <g style="fill: #222222" transform="translate(506.526525 223.565513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_72">+ <!-- 95.68 -->+ <g style="fill: #222222" transform="translate(595.806525 223.565513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_73">+ <!-- 82.50 -->+ <g style="fill: #222222" transform="translate(685.086525 223.565513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_74">+ <!-- 80.62 -->+ <g style="fill: #222222" transform="translate(774.366525 223.565513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_75">+ <!-- 56.14 -->+ <g style="fill: #222222" transform="translate(863.646525 223.565513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_76">+ <!-- 66.92 -->+ <g style="fill: #222222" transform="translate(952.926525 223.565513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_77">+ <!-- 67.27 -->+ <g style="fill: #222222" transform="translate(1041.269288 223.565513) scale(0.1584 -0.1584)">+ <defs>+ <path id="DejaVuSans-Bold-1a" d="M 428 4666+L 3944 4666+L 3944 3988+L 2125 0+L 953 0+L 2675 3781+L 428 3781+L 428 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-Bold-19"/>+ <use xlink:href="#DejaVuSans-Bold-1a" transform="translate(69.578125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-11" transform="translate(139.15625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-15" transform="translate(177.140625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-1a" transform="translate(246.71875 0)"/>+ </g>+ </g>+ <g id="text_78">+ <!-- BF16 KV -->+ <g style="fill: #222222" transform="translate(198 245.623444) scale(0.1512 -0.1512)">+ <use xlink:href="#DejaVuSans-25"/>+ <use xlink:href="#DejaVuSans-29" transform="translate(68.609375 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(126.125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(189.75 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(253.375 0)"/>+ <use xlink:href="#DejaVuSans-2e" transform="translate(285.15625 0)"/>+ <use xlink:href="#DejaVuSans-39" transform="translate(350.734375 0)"/>+ </g>+ </g>+ <g id="text_79">+ <!-- 60.83 -->+ <g style="fill: #222222" transform="translate(417.246525 245.885513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_80">+ <!-- 43.41 -->+ <g style="fill: #222222" transform="translate(506.526525 245.885513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-17"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_81">+ <!-- 98.39 -->+ <g style="fill: #222222" transform="translate(595.806525 245.885513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_82">+ <!-- 90.62 -->+ <g style="fill: #222222" transform="translate(685.086525 245.885513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_83">+ <!-- 83.54 -->+ <g style="fill: #222222" transform="translate(774.366525 245.885513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_84">+ <!-- 64.00 -->+ <g style="fill: #222222" transform="translate(863.646525 245.885513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_85">+ <!-- 71.28 -->+ <g style="fill: #222222" transform="translate(952.926525 245.885513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_86">+ <!-- 73.15 -->+ <g style="fill: #222222" transform="translate(1043.646525 245.885513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_87">+ <!-- FP8 KV -->+ <g style="fill: #222222" transform="translate(198 267.943444) scale(0.1512 -0.1512)">+ <use xlink:href="#DejaVuSans-29"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(57.515625 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(117.8125 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(181.4375 0)"/>+ <use xlink:href="#DejaVuSans-2e" transform="translate(213.21875 0)"/>+ <use xlink:href="#DejaVuSans-39" transform="translate(278.796875 0)"/>+ </g>+ </g>+ <g id="text_88">+ <!-- 58.45 -->+ <g style="fill: #222222" transform="translate(417.246525 268.205513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_89">+ <!-- 40.60 -->+ <g style="fill: #222222" transform="translate(506.526525 268.205513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-17"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_90">+ <!-- 98.03 -->+ <g style="fill: #222222" transform="translate(595.806525 268.205513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_91">+ <!-- 91.56 -->+ <g style="fill: #222222" transform="translate(685.086525 268.205513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_92">+ <!-- 82.81 -->+ <g style="fill: #222222" transform="translate(774.366525 268.205513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_93">+ <!-- 60.86 -->+ <g style="fill: #222222" transform="translate(863.646525 268.205513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_94">+ <!-- 70.58 -->+ <g style="fill: #222222" transform="translate(952.926525 268.205513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_95">+ <!-- 71.84 -->+ <g style="fill: #222222" transform="translate(1043.646525 268.205513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_96">+ <!-- TurboQuant-4bit-nc -->+ <g style="fill: #222222" transform="translate(198 289.529888) scale(0.1512 -0.1512)">+ <use xlink:href="#DejaVuSans-37"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(45.890625 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(109.265625 0)"/>+ <use xlink:href="#DejaVuSans-45" transform="translate(150.375 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(213.859375 0)"/>+ <use xlink:href="#DejaVuSans-34" transform="translate(275.046875 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(353.765625 0)"/>+ <use xlink:href="#DejaVuSans-44" transform="translate(417.140625 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(478.421875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(541.796875 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(581 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(617.078125 0)"/>+ <use xlink:href="#DejaVuSans-45" transform="translate(680.703125 0)"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(744.1875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(771.96875 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(811.171875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(847.25 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(910.625 0)"/>+ </g>+ </g>+ <g id="text_97">+ <!-- 59.56 -->+ <g style="fill: #222222" transform="translate(417.246525 290.525513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_98">+ <!-- 42.22 -->+ <g style="fill: #222222" transform="translate(506.526525 290.638125) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-17"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_99">+ <!-- 98.02 -->+ <g style="fill: #222222" transform="translate(595.806525 290.525513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_100">+ <!-- 88.65 -->+ <g style="fill: #222222" transform="translate(685.086525 290.525513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_101">+ <!-- 83.85 -->+ <g style="fill: #222222" transform="translate(774.366525 290.525513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_102">+ <!-- 56.57 -->+ <g style="fill: #222222" transform="translate(863.646525 290.525513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_103">+ <!-- 70.64 -->+ <g style="fill: #222222" transform="translate(952.926525 290.525513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_104">+ <!-- 71.36 -->+ <g style="fill: #222222" transform="translate(1043.646525 290.525513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_105">+ <!-- SAW-INT4 -->+ <g style="fill: #222222" transform="translate(198 312.583444) scale(0.1512 -0.1512)">+ <use xlink:href="#DejaVuSans-36"/>+ <use xlink:href="#DejaVuSans-24" transform="translate(65.34375 0)"/>+ <use xlink:href="#DejaVuSans-3a" transform="translate(128.28125 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(223.109375 0)"/>+ <use xlink:href="#DejaVuSans-2c" transform="translate(259.1875 0)"/>+ <use xlink:href="#DejaVuSans-31" transform="translate(288.6875 0)"/>+ <use xlink:href="#DejaVuSans-37" transform="translate(363.5 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(424.578125 0)"/>+ </g>+ </g>+ <g id="text_106">+ <!-- 57.27 -->+ <g style="fill: #222222" transform="translate(417.246525 312.845513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_107">+ <!-- 36.31 -->+ <g style="fill: #222222" transform="translate(506.526525 312.845513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_108">+ <!-- 97.58 -->+ <g style="fill: #222222" transform="translate(595.806525 312.845513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_109">+ <!-- 88.85 -->+ <g style="fill: #222222" transform="translate(685.086525 312.845513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_110">+ <!-- 83.75 -->+ <g style="fill: #222222" transform="translate(774.366525 312.845513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_111">+ <!-- 57.71 -->+ <g style="fill: #222222" transform="translate(863.646525 312.741563) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_112">+ <!-- 70.39 -->+ <g style="fill: #222222" transform="translate(952.926525 312.845513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_113">+ <!-- 70.27 -->+ <g style="fill: #222222" transform="translate(1043.646525 312.845513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_114">+ <!-- TRT-LLM-NVFP4 -->+ <g style="fill: #222222" transform="translate(198 334.911713) scale(0.1512 -0.1512)">+ <use xlink:href="#DejaVuSans-37"/>+ <use xlink:href="#DejaVuSans-35" transform="translate(61.078125 0)"/>+ <use xlink:href="#DejaVuSans-37" transform="translate(123.28125 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(175.1875 0)"/>+ <use xlink:href="#DejaVuSans-2f" transform="translate(211.265625 0)"/>+ <use xlink:href="#DejaVuSans-2f" transform="translate(266.984375 0)"/>+ <use xlink:href="#DejaVuSans-30" transform="translate(322.703125 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(408.984375 0)"/>+ <use xlink:href="#DejaVuSans-31" transform="translate(445.0625 0)"/>+ <use xlink:href="#DejaVuSans-39" transform="translate(519.875 0)"/>+ <use xlink:href="#DejaVuSans-29" transform="translate(588.28125 0)"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(645.796875 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(706.09375 0)"/>+ </g>+ </g>+ <g id="text_115">+ <!-- 50.72 -->+ <g style="fill: #222222" transform="translate(417.246525 335.165513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_116">+ <!-- 28.41 -->+ <g style="fill: #222222" transform="translate(506.526525 335.165513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-15"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_117">+ <!-- 89.42 -->+ <g style="fill: #222222" transform="translate(595.806525 335.165513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_118">+ <!-- 88.54 -->+ <g style="fill: #222222" transform="translate(685.086525 335.165513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_119">+ <!-- 81.77 -->+ <g style="fill: #222222" transform="translate(774.366525 335.165513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_120">+ <!-- 61.00 -->+ <g style="fill: #222222" transform="translate(863.646525 335.165513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_121">+ <!-- 70.90 -->+ <g style="fill: #222222" transform="translate(952.926525 335.165513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_122">+ <!-- 67.25 -->+ <g style="fill: #222222" transform="translate(1043.646525 335.165513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_123">+ <!-- BitDecoding-MXFP4 -->+ <g style="fill: #222222" transform="translate(198 355.892175) scale(0.1512 -0.1512)">+ <use xlink:href="#DejaVuSans-25"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(68.609375 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(96.390625 0)"/>+ <use xlink:href="#DejaVuSans-27" transform="translate(135.59375 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(212.59375 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(274.125 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(329.109375 0)"/>+ <use xlink:href="#DejaVuSans-47" transform="translate(390.296875 0)"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(453.78125 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(481.5625 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(544.9375 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(608.421875 0)"/>+ <use xlink:href="#DejaVuSans-30" transform="translate(644.5 0)"/>+ <use xlink:href="#DejaVuSans-3b" transform="translate(730.78125 0)"/>+ <use xlink:href="#DejaVuSans-29" transform="translate(799.28125 0)"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(856.796875 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(917.09375 0)"/>+ </g>+ </g>+ <g id="text_124">+ <!-- 58.53 -->+ <g style="fill: #222222" transform="translate(417.246525 357.485513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_125">+ <!-- 42.71 -->+ <g style="fill: #222222" transform="translate(506.526525 357.598125) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-17"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_126">+ <!-- 98.33 -->+ <g style="fill: #222222" transform="translate(595.806525 357.485513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_127">+ <!-- 91.04 -->+ <g style="fill: #222222" transform="translate(685.086525 357.485513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_128">+ <!-- 84.58 -->+ <g style="fill: #222222" transform="translate(774.366525 357.485513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_129">+ <!-- 62.29 -->+ <g style="fill: #222222" transform="translate(863.646525 357.485513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1c" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_130">+ <!-- 70.71 -->+ <g style="fill: #222222" transform="translate(952.926525 357.485513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_131">+ <!-- 72.60 -->+ <g style="fill: #222222" transform="translate(1043.646525 357.485513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_132">+ <!-- BitDecoding-NVFP4 -->+ <g style="fill: #222222" transform="translate(198 378.153113) scale(0.1512 -0.1512)">+ <use xlink:href="#DejaVuSans-Bold-25"/>+ <use xlink:href="#DejaVuSans-Bold-4c" transform="translate(76.21875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-57" transform="translate(110.5 0)"/>+ <use xlink:href="#DejaVuSans-Bold-27" transform="translate(158.296875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-48" transform="translate(241.3125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-46" transform="translate(309.140625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-52" transform="translate(368.421875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-47" transform="translate(437.125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-4c" transform="translate(508.703125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-51" transform="translate(542.984375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-4a" transform="translate(614.171875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-10" transform="translate(685.75 0)"/>+ <use xlink:href="#DejaVuSans-Bold-31" transform="translate(727.25 0)"/>+ <use xlink:href="#DejaVuSans-Bold-39" transform="translate(810.9375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-29" transform="translate(888.328125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-33" transform="translate(956.640625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-17" transform="translate(1029.9375 0)"/>+ </g>+ </g>+ <g id="text_133">+ <!-- 58.86 -->+ <g style="fill: #222222" transform="translate(417.246525 379.805513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_134">+ <!-- 42.22 -->+ <g style="fill: #222222" transform="translate(506.526525 379.918125) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-17"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_135">+ <!-- 98.58 -->+ <g style="fill: #222222" transform="translate(595.806525 379.805513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_136">+ <!-- 90.83 -->+ <g style="fill: #222222" transform="translate(685.086525 379.805513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1c"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_137">+ <!-- 85.31 -->+ <g style="fill: #222222" transform="translate(774.366525 379.805513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_138">+ <!-- 62.57 -->+ <g style="fill: #222222" transform="translate(863.646525 379.805513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_139">+ <!-- 70.83 -->+ <g style="fill: #222222" transform="translate(952.926525 379.805513) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-1a"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(159.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(222.65625 0)"/>+ </g>+ </g>+ <g id="text_140">+ <!-- 72.74 -->+ <g style="fill: #222222" transform="translate(1041.269288 379.918125) scale(0.1584 -0.1584)">+ <use xlink:href="#DejaVuSans-Bold-1a"/>+ <use xlink:href="#DejaVuSans-Bold-15" transform="translate(69.578125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-11" transform="translate(139.15625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-1a" transform="translate(177.140625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-17" transform="translate(246.71875 0)"/>+ </g>+ </g>+ <g id="text_141">+ <!-- Qwen3-4B- -->+ <g style="fill: #222222" transform="translate(72.641306 144.892406) scale(0.1512 -0.1512)">+ <use xlink:href="#DejaVuSans-34"/>+ <use xlink:href="#DejaVuSans-5a" transform="translate(78.71875 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(160.5 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(222.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(285.40625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(349.03125 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(385.109375 0)"/>+ <use xlink:href="#DejaVuSans-25" transform="translate(448.734375 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(517.34375 0)"/>+ </g>+ <!-- Thinking-2507 -->+ <g style="fill: #222222" transform="translate(60.143681 165.596175) scale(0.1512 -0.1512)">+ <defs>+ <path id="DejaVuSans-4e" d="M 581 4863+L 1159 4863+L 1159 1991+L 2875 3500+L 3609 3500+L 1753 1863+L 3688 0+L 2938 0+L 1159 1709+L 1159 0+L 581 0+L 581 4863+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-37"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(61.078125 0)"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(124.453125 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(152.234375 0)"/>+ <use xlink:href="#DejaVuSans-4e" transform="translate(215.609375 0)"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(273.515625 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(301.296875 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(364.671875 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(428.15625 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(464.234375 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(527.859375 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(591.484375 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(655.109375 0)"/>+ </g>+ </g>+ <g id="text_142">+ <!-- Qwen3-30B-A3B- -->+ <g style="fill: #222222" transform="translate(53.1675 301.415625) scale(0.144 -0.144)">+ <use xlink:href="#DejaVuSans-34"/>+ <use xlink:href="#DejaVuSans-5a" transform="translate(78.71875 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(160.5 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(222.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(285.40625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(349.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(385.109375 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(448.734375 0)"/>+ <use xlink:href="#DejaVuSans-25" transform="translate(512.359375 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(580.96875 0)"/>+ <use xlink:href="#DejaVuSans-24" transform="translate(614.84375 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(683.25 0)"/>+ <use xlink:href="#DejaVuSans-25" transform="translate(746.875 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(815.484375 0)"/>+ </g>+ <!-- Thinking-2507 -->+ <g style="fill: #222222" transform="translate(62.731125 321.1335) scale(0.144 -0.144)">+ <use xlink:href="#DejaVuSans-37"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(61.078125 0)"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(124.453125 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(152.234375 0)"/>+ <use xlink:href="#DejaVuSans-4e" transform="translate(215.609375 0)"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(273.515625 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(301.296875 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(364.671875 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(428.15625 0)"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(464.234375 0)"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(527.859375 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(591.484375 0)"/>+ <use xlink:href="#DejaVuSans-1a" transform="translate(655.109375 0)"/>+ </g>+ </g>+ <g id="text_143">+ <!-- Accuracy (%). Avg. is the mean across seven benchmarks. -->+ <g style="fill: #555555" transform="translate(39.6 412.734825) scale(0.1368 -0.1368)">+ <defs>+ <path id="DejaVuSans-5c" d="M 2059 -325+Q 1816 -950 1584 -1140+Q 1353 -1331 966 -1331+L 506 -1331+L 506 -850+L 844 -850+Q 1081 -850 1212 -737+Q 1344 -625 1503 -206+L 1606 56+L 191 3500+L 800 3500+L 1894 763+L 2988 3500+L 3597 3500+L 2059 -325+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-b" d="M 1984 4856+Q 1566 4138 1362 3434+Q 1159 2731 1159 2009+Q 1159 1288 1364 580+Q 1569 -128 1984 -844+L 1484 -844+Q 1016 -109 783 600+Q 550 1309 550 2009+Q 550 2706 781 3412+Q 1013 4119 1484 4856+L 1984 4856+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-8" d="M 4653 2053+Q 4381 2053 4226 1822+Q 4072 1591 4072 1178+Q 4072 772 4226 539+Q 4381 306 4653 306+Q 4919 306 5073 539+Q 5228 772 5228 1178+Q 5228 1588 5073 1820+Q 4919 2053 4653 2053+z+M 4653 2450+Q 5147 2450 5437 2106+Q 5728 1763 5728 1178+Q 5728 594 5436 251+Q 5144 -91 4653 -91+Q 4153 -91 3862 251+Q 3572 594 3572 1178+Q 3572 1766 3864 2108+Q 4156 2450 4653 2450+z+M 1428 4353+Q 1159 4353 1004 4120+Q 850 3888 850 3481+Q 850 3069 1003 2837+Q 1156 2606 1428 2606+Q 1700 2606 1854 2837+Q 2009 3069 2009 3481+Q 2009 3884 1853 4118+Q 1697 4353 1428 4353+z+M 4250 4750+L 4750 4750+L 1831 -91+L 1331 -91+L 4250 4750+z+M 1428 4750+Q 1922 4750 2215 4408+Q 2509 4066 2509 3481+Q 2509 2891 2217 2550+Q 1925 2209 1428 2209+Q 931 2209 642 2551+Q 353 2894 353 3481+Q 353 4063 643 4406+Q 934 4750 1428 4750+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-c" d="M 513 4856+L 1013 4856+Q 1481 4119 1714 3412+Q 1947 2706 1947 2009+Q 1947 1309 1714 600+Q 1481 -109 1013 -844+L 513 -844+Q 928 -128 1133 580+Q 1338 1288 1338 2009+Q 1338 2731 1133 3434+Q 928 4138 513 4856+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-59" d="M 191 3500+L 800 3500+L 1894 563+L 2988 3500+L 3597 3500+L 2284 0+L 1503 0+L 191 3500+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-56" d="M 2834 3397+L 2834 2853+Q 2591 2978 2328 3040+Q 2066 3103 1784 3103+Q 1356 3103 1142 2972+Q 928 2841 928 2578+Q 928 2378 1081 2264+Q 1234 2150 1697 2047+L 1894 2003+Q 2506 1872 2764 1633+Q 3022 1394 3022 966+Q 3022 478 2636 193+Q 2250 -91 1575 -91+Q 1294 -91 989 -36+Q 684 19 347 128+L 347 722+Q 666 556 975 473+Q 1284 391 1588 391+Q 1994 391 2212 530+Q 2431 669 2431 922+Q 2431 1156 2273 1281+Q 2116 1406 1581 1522+L 1381 1569+Q 847 1681 609 1914+Q 372 2147 372 2553+Q 372 3047 722 3315+Q 1072 3584 1716 3584+Q 2034 3584 2315 3537+Q 2597 3491 2834 3397+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-50" d="M 3328 2828+Q 3544 3216 3844 3400+Q 4144 3584 4550 3584+Q 5097 3584 5394 3201+Q 5691 2819 5691 2113+L 5691 0+L 5113 0+L 5113 2094+Q 5113 2597 4934 2840+Q 4756 3084 4391 3084+Q 3944 3084 3684 2787+Q 3425 2491 3425 1978+L 3425 0+L 2847 0+L 2847 2094+Q 2847 2600 2669 2842+Q 2491 3084 2119 3084+Q 1678 3084 1418 2786+Q 1159 2488 1159 1978+L 1159 0+L 581 0+L 581 3500+L 1159 3500+L 1159 2956+Q 1356 3278 1631 3431+Q 1906 3584 2284 3584+Q 2666 3584 2933 3390+Q 3200 3197 3328 2828+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-24"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(66.65625 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(121.640625 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(176.625 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(240 0)"/>+ <use xlink:href="#DejaVuSans-44" transform="translate(281.109375 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(342.390625 0)"/>+ <use xlink:href="#DejaVuSans-5c" transform="translate(397.375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(456.5625 0)"/>+ <use xlink:href="#DejaVuSans-b" transform="translate(488.34375 0)"/>+ <use xlink:href="#DejaVuSans-8" transform="translate(527.359375 0)"/>+ <use xlink:href="#DejaVuSans-c" transform="translate(622.375 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(661.390625 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(693.171875 0)"/>+ <use xlink:href="#DejaVuSans-24" transform="translate(724.953125 0)"/>+ <use xlink:href="#DejaVuSans-59" transform="translate(787.5 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(846.6875 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(910.171875 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(941.953125 0)"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(973.734375 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1001.515625 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(1053.609375 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(1085.390625 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(1124.59375 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(1187.96875 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(1249.5 0)"/>+ <use xlink:href="#DejaVuSans-50" transform="translate(1281.28125 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(1378.6875 0)"/>+ <use xlink:href="#DejaVuSans-44" transform="translate(1440.21875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(1501.5 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(1564.875 0)"/>+ <use xlink:href="#DejaVuSans-44" transform="translate(1596.65625 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(1657.9375 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(1712.921875 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(1751.828125 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1813.015625 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1865.109375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(1917.203125 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1948.984375 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(2001.078125 0)"/>+ <use xlink:href="#DejaVuSans-59" transform="translate(2062.609375 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(2121.796875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(2183.328125 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(2246.703125 0)"/>+ <use xlink:href="#DejaVuSans-45" transform="translate(2278.484375 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(2341.96875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(2403.5 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(2466.875 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(2521.859375 0)"/>+ <use xlink:href="#DejaVuSans-50" transform="translate(2585.234375 0)"/>+ <use xlink:href="#DejaVuSans-44" transform="translate(2682.640625 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(2743.921875 0)"/>+ <use xlink:href="#DejaVuSans-4e" transform="translate(2785.03125 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(2842.9375 0)"/>+ <use xlink:href="#DejaVuSans-11" transform="translate(2895.03125 0)"/>+ </g>+ </g>+ <g id="text_144">+ <!-- LB-Pro: LongBench Pro · RULER: RULER-NIAH · LCB v6: LiveCodeBench v6 · GPQA: GPQA-Diamond -->+ <g style="fill: #555555" transform="translate(39.6 432.894825) scale(0.1368 -0.1368)">+ <defs>+ <path id="DejaVuSans-1d" d="M 750 794+L 1409 794+L 1409 0+L 750 0+L 750 794+z+M 750 3309+L 1409 3309+L 1409 2516+L 750 2516+L 750 3309+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-79" d="M 684 2619+L 1344 2619+L 1344 1825+L 684 1825+L 684 2619+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-38" d="M 556 4666+L 1191 4666+L 1191 1831+Q 1191 1081 1462 751+Q 1734 422 2344 422+Q 2950 422 3222 751+Q 3494 1081 3494 1831+L 3494 4666+L 4128 4666+L 4128 1753+Q 4128 841 3676 375+Q 3225 -91 2344 -91+Q 1459 -91 1007 375+Q 556 841 556 1753+L 556 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-28" d="M 628 4666+L 3578 4666+L 3578 4134+L 1259 4134+L 1259 2753+L 3481 2753+L 3481 2222+L 1259 2222+L 1259 531+L 3634 531+L 3634 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-2b" d="M 628 4666+L 1259 4666+L 1259 2753+L 3553 2753+L 3553 4666+L 4184 4666+L 4184 0+L 3553 0+L 3553 2222+L 1259 2222+L 1259 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-2a" d="M 3809 666+L 3809 1919+L 2778 1919+L 2778 2438+L 4434 2438+L 4434 434+Q 4069 175 3628 42+Q 3188 -91 2688 -91+Q 1594 -91 976 548+Q 359 1188 359 2328+Q 359 3472 976 4111+Q 1594 4750 2688 4750+Q 3144 4750 3555 4637+Q 3966 4525 4313 4306+L 4313 3634+Q 3963 3931 3569 4081+Q 3175 4231 2741 4231+Q 1884 4231 1454 3753+Q 1025 3275 1025 2328+Q 1025 1384 1454 906+Q 1884 428 2741 428+Q 3075 428 3337 486+Q 3600 544 3809 666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-2f"/>+ <use xlink:href="#DejaVuSans-25" transform="translate(55.71875 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(124.328125 0)"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(160.40625 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(218.953125 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(257.859375 0)"/>+ <use xlink:href="#DejaVuSans-1d" transform="translate(319.046875 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(352.734375 0)"/>+ <use xlink:href="#DejaVuSans-2f" transform="translate(384.515625 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(438.484375 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(499.671875 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(563.046875 0)"/>+ <use xlink:href="#DejaVuSans-25" transform="translate(626.53125 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(695.140625 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(756.671875 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(820.046875 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(875.03125 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(938.40625 0)"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(970.1875 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(1028.734375 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(1067.640625 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(1128.828125 0)"/>+ <use xlink:href="#DejaVuSans-79" transform="translate(1160.609375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(1192.390625 0)"/>+ <use xlink:href="#DejaVuSans-35" transform="translate(1224.171875 0)"/>+ <use xlink:href="#DejaVuSans-38" transform="translate(1293.65625 0)"/>+ <use xlink:href="#DejaVuSans-2f" transform="translate(1366.84375 0)"/>+ <use xlink:href="#DejaVuSans-28" transform="translate(1422.5625 0)"/>+ <use xlink:href="#DejaVuSans-35" transform="translate(1485.75 0)"/>+ <use xlink:href="#DejaVuSans-1d" transform="translate(1552.15625 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(1585.84375 0)"/>+ <use xlink:href="#DejaVuSans-35" transform="translate(1617.625 0)"/>+ <use xlink:href="#DejaVuSans-38" transform="translate(1687.109375 0)"/>+ <use xlink:href="#DejaVuSans-2f" transform="translate(1760.296875 0)"/>+ <use xlink:href="#DejaVuSans-28" transform="translate(1816.015625 0)"/>+ <use xlink:href="#DejaVuSans-35" transform="translate(1879.203125 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(1944.640625 0)"/>+ <use xlink:href="#DejaVuSans-31" transform="translate(1980.71875 0)"/>+ <use xlink:href="#DejaVuSans-2c" transform="translate(2055.53125 0)"/>+ <use xlink:href="#DejaVuSans-24" transform="translate(2085.03125 0)"/>+ <use xlink:href="#DejaVuSans-2b" transform="translate(2153.4375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(2228.640625 0)"/>+ <use xlink:href="#DejaVuSans-79" transform="translate(2260.421875 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(2292.203125 0)"/>+ <use xlink:href="#DejaVuSans-2f" transform="translate(2323.984375 0)"/>+ <use xlink:href="#DejaVuSans-26" transform="translate(2379.703125 0)"/>+ <use xlink:href="#DejaVuSans-25" transform="translate(2449.53125 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(2518.140625 0)"/>+ <use xlink:href="#DejaVuSans-59" transform="translate(2549.921875 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(2609.109375 0)"/>+ <use xlink:href="#DejaVuSans-1d" transform="translate(2672.734375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(2706.421875 0)"/>+ <use xlink:href="#DejaVuSans-2f" transform="translate(2738.203125 0)"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(2793.921875 0)"/>+ <use xlink:href="#DejaVuSans-59" transform="translate(2821.703125 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(2880.890625 0)"/>+ <use xlink:href="#DejaVuSans-26" transform="translate(2942.421875 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(3012.25 0)"/>+ <use xlink:href="#DejaVuSans-47" transform="translate(3073.4375 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(3136.921875 0)"/>+ <use xlink:href="#DejaVuSans-25" transform="translate(3198.453125 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(3267.0625 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(3328.59375 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(3391.96875 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(3446.953125 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(3510.328125 0)"/>+ <use xlink:href="#DejaVuSans-59" transform="translate(3542.109375 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(3601.296875 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(3664.921875 0)"/>+ <use xlink:href="#DejaVuSans-79" transform="translate(3696.703125 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(3728.484375 0)"/>+ <use xlink:href="#DejaVuSans-2a" transform="translate(3760.265625 0)"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(3837.75 0)"/>+ <use xlink:href="#DejaVuSans-34" transform="translate(3898.046875 0)"/>+ <use xlink:href="#DejaVuSans-24" transform="translate(3976.765625 0)"/>+ <use xlink:href="#DejaVuSans-1d" transform="translate(4043.421875 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(4077.109375 0)"/>+ <use xlink:href="#DejaVuSans-2a" transform="translate(4108.890625 0)"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(4186.375 0)"/>+ <use xlink:href="#DejaVuSans-34" transform="translate(4246.671875 0)"/>+ <use xlink:href="#DejaVuSans-24" transform="translate(4325.390625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(4391.59375 0)"/>+ <use xlink:href="#DejaVuSans-27" transform="translate(4427.671875 0)"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(4504.671875 0)"/>+ <use xlink:href="#DejaVuSans-44" transform="translate(4532.453125 0)"/>+ <use xlink:href="#DejaVuSans-50" transform="translate(4593.734375 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(4691.140625 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(4752.328125 0)"/>+ <use xlink:href="#DejaVuSans-47" transform="translate(4815.703125 0)"/>+ </g>+ </g>+ </g>+ </g>+ <defs>+ <clipPath id="pb7bde0cdf1">+ <rect x="0" y="0" width="1152" height="446.4"/>+ </clipPath>+ </defs>+</svg>diff --git a/imgs/e2e-serving-throughput.svg b/imgs/e2e-serving-throughput.svgnew file mode 100644index 0000000..8faeb54--- /dev/null+++ b/imgs/e2e-serving-throughput.svg@@ -0,0 +1,3864 @@+<?xml version="1.0" encoding="utf-8" standalone="no"?>+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">+<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="993.6pt" height="770.4pt" viewBox="0 0 993.6 770.4" xmlns="http://www.w3.org/2000/svg" version="1.1">+<desc>{"source": "Paper end-to-end serving throughput grids", "gpu": "NVIDIA B300", "concurrency": [1, 8, 16, 32, 64], "estimator": "matched-run per-GPU and per-request decode rates, including queueing", "workloads": {"64k": {"tensor_parallel_size": 1, "models": {"Qwen3-4B": {"bf16": {"per_gpu": [241.5924375575299, 592.0546080792789, 659.8624128146297, 602.4492459548104, 619.6951177625699], "per_request": [241.5924375575299, 74.00682600990986, 41.241400800914356, 18.826538936087825, 9.682736215040155], "exceeds_kv_capacity": [0, 0, 0, 1, 1]}, "fp8kv": {"per_gpu": [275.6612100891455, 960.1350494719957, 1153.8267785359044, 1250.9347629851327, 1261.6795026627674], "per_request": [275.6612100891455, 120.01688118399946, 72.11417365849402, 39.0917113432854, 19.71374222910574], "exceeds_kv_capacity": [0, 0, 0, 0, 1]}, "turboquant": {"per_gpu": [33.77721200975613, 85.51637763052268, 92.01574842326971, 95.58252368644568, 97.1684250161144], "per_request": [33.77721200975613, 10.689547203815335, 5.750984276454357, 2.9869538652014276, 1.5182566408767875], "exceeds_kv_capacity": [0, 0, 0, 0, 0]}, "trtllm-nvfp4": {"per_gpu": [285.37237758588265, 1088.7995844845398, 1372.3749546394888, 1555.7950816314824, 1728.4975904187527], "per_request": [285.37237758588265, 136.09994806056747, 85.77343466496805, 48.618596300983825, 27.00777485029301], "exceeds_kv_capacity": [0, 0, 0, 0, 0]}, "bitdecoding-nvfp4": {"per_gpu": [317.6038344336229, 1235.584798859373, 1585.4663884966283, 1823.4871331830989, 1951.9075490671999], "per_request": [317.6038344336229, 154.44809985742162, 99.09164928103927, 56.98397291197184, 30.498555454174998], "exceeds_kv_capacity": [0, 0, 0, 0, 0]}}, "Qwen3-30B-A3B": {"bf16": {"per_gpu": [228.56547027633744, 631.1492528074868, 770.4167061835207, 750.5783582631608, 784.9234270870667], "per_request": [228.56547027633744, 78.89365660093586, 48.15104413647004, 23.455573695723775, 12.264428548235417], "exceeds_kv_capacity": [0, 0, 0, 1, 1]}, "fp8kv": {"per_gpu": [250.0380817750774, 871.8636109216527, 1159.2780712997844, 1442.1881630856462, 1368.764439807491], "per_request": [250.0380817750774, 108.98295136520659, 72.45487945623653, 45.068380096426445, 21.386944371992048], "exceeds_kv_capacity": [0, 0, 0, 0, 1]}, "turboquant": {"per_gpu": [26.13314504462074, 64.0818538096659, 69.02244599303239, 72.01753487622298, 73.54122900376797], "per_request": [26.13314504462074, 8.010231726208238, 4.3139028745645245, 2.250547964881968, 1.1490817031838745], "exceeds_kv_capacity": [0, 0, 0, 0, 0]}, "trtllm-nvfp4": {"per_gpu": [243.91946226054782, 940.4192075878019, 1269.8402859694327, 1660.4325443036703, 2005.2974846104082], "per_request": [243.91946226054782, 117.55240094847524, 79.36501787308954, 51.888517009489696, 31.33277319703763], "exceeds_kv_capacity": [0, 0, 0, 0, 0]}, "bitdecoding-nvfp4": {"per_gpu": [272.8877523887702, 949.1173923392904, 1363.6021749780762, 1721.1247467720766, 2073.429162010721], "per_request": [272.8877523887702, 118.6396740424113, 85.22513593612976, 53.785148336627394, 32.397330656417516], "exceeds_kv_capacity": [0, 0, 0, 0, 0]}}}}, "128k": {"tensor_parallel_size": 2, "models": {"Qwen3-4B": {"bf16": {"per_gpu": [124.1973595430094, 299.4519049542983, 333.8634667713076, 326.7250446660864, 337.7389609327248], "per_request": [248.3947190860188, 74.86297623857457, 41.73293334641345, 20.4203152916304, 10.55434252914765], "exceeds_kv_capacity": [0, 0, 0, 1, 1]}, "fp8kv": {"per_gpu": [145.87523409155256, 494.7357757341027, 583.0453731112084, 648.7895934747614, 604.110584005483], "per_request": [291.7504681831051, 123.68394393352567, 72.88067163890105, 40.54934959217259, 18.878455750171344], "exceeds_kv_capacity": [0, 0, 0, 0, 1]}, "turboquant": {"per_gpu": [9.676069698317876, 40.23924872011958, 43.886439080731215, 46.588597463866165, 47.74506966464605], "per_request": [19.352139396635753, 10.059812180029896, 5.485804885091402, 2.9117873414916353, 1.4920334270201892], "exceeds_kv_capacity": [0, 0, 0, 0, 0]}, "trtllm-nvfp4": {"per_gpu": [147.72375412421914, 557.7098355060464, 690.1414783515653, 799.1748616551005, 857.7561091195889], "per_request": [295.4475082484383, 139.4274588765116, 86.26768479394566, 49.94842885344378, 26.804878409987154], "exceeds_kv_capacity": [0, 0, 0, 0, 0]}, "bitdecoding-nvfp4": {"per_gpu": [151.88388555845438, 642.5813721535894, 801.8377432831151, 946.5013086672922, 1011.9989727829321], "per_request": [303.76777111690876, 160.64534303839736, 100.22971791038938, 59.15633179170576, 31.62496789946663], "exceeds_kv_capacity": [0, 0, 0, 0, 0]}}, "Qwen3-30B-A3B": {"bf16": {"per_gpu": [112.54071157576458, 350.1778371409967, 420.8408846574361, 473.16144702199017, 434.4186636424773], "per_request": [225.08142315152915, 87.54445928524918, 52.60511058217951, 29.572590438874386, 13.575583238827416], "exceeds_kv_capacity": [0, 0, 0, 0, 1]}, "fp8kv": {"per_gpu": [124.51018487691535, 490.5913143194946, 654.4103061094695, 793.1147453943487, 906.0349142430415], "per_request": [249.0203697538307, 122.64782857987365, 81.80128826368369, 49.56967158714679, 28.313591070095047], "exceeds_kv_capacity": [0, 0, 0, 0, 0]}, "turboquant": {"per_gpu": [7.413901840193434, 30.646106213700516, 33.24506107772419, 35.18304348492216, 36.18739289553637], "per_request": [14.827803680386868, 7.661526553425129, 4.1556326347155235, 2.198940217807635, 1.1308560279855115], "exceeds_kv_capacity": [0, 0, 0, 0, 0]}, "trtllm-nvfp4": {"per_gpu": [122.43710158716446, 556.0613079564367, 741.7475910870561, 931.072730409484, 1098.5401050340533], "per_request": [244.87420317432893, 139.01532698910918, 92.71844888588201, 58.19204565059275, 34.32937828231417], "exceeds_kv_capacity": [0, 0, 0, 0, 0]}, "bitdecoding-nvfp4": {"per_gpu": [133.8052221086704, 564.6055827025602, 765.2116443723951, 965.0420444038357, 1136.4808844893296], "per_request": [267.6104442173408, 141.15139567564006, 95.65145554654939, 60.31512777523973, 35.51502764029155], "exceeds_kv_capacity": [0, 0, 0, 0, 0]}}}}}}</desc>+ <title>End-to-end serving throughput on NVIDIA B300</title>+ <metadata>+ <rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">+ <cc:Work>+ <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>+ <dc:title>End-to-end serving throughput on NVIDIA B300</dc:title>+ <dc:description>Paper throughput results in two rows: 64K input / 2K output at TP=1 and 128K input / 2K output at TP=2. Each row compares Qwen3-4B and Qwen3-30B-A3B. Hollow markers indicate exceeded KV capacity. Rates include queueing; TurboQuant uses 100 output tokens and a 32-token baseline, other methods use 2048 output tokens and a 128-token baseline.</dc:description>+ <dc:format>image/svg+xml</dc:format>+ <dc:creator>+ <cc:Agent>+ <dc:title>BitDecoding-FP4</dc:title>+ </cc:Agent>+ </dc:creator>+ </cc:Work>+ </rdf:RDF>+ </metadata>+ <defs>+ <style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style>+ </defs>+ <g id="figure_1">+ <g id="patch_1">+ <path d="M 0 770.4+L 993.6 770.4+L 993.6 0+L 0 0+z+" style="fill: #ffffff"/>+ </g>+ <g id="axes_1">+ <g id="patch_2">+ <path d="M 84.456 350.532+L 466.992 350.532+L 466.992 138.672+L 84.456 138.672+z+" style="fill: #ffffff"/>+ </g>+ <g id="matplotlib.axis_1">+ <g id="xtick_1">+ <g id="line2d_1">+ <path d="M 84.456 350.532+L 84.456 138.672+" clip-path="url(#p4105837502)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_2">+ <defs>+ <path id="m328dabad00" d="M 0 0+L 0 4+" style="stroke: #444444; stroke-width: 0.8"/>+ </defs>+ <g>+ <use xlink:href="#m328dabad00" x="84.456" y="350.532" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_1">+ <!-- 0 -->+ <g style="fill: #444444" transform="translate(80.320375 369.408953) scale(0.13 -0.13)">+ <defs>+ <path id="DejaVuSans-13" d="M 2034 4250+Q 1547 4250 1301 3770+Q 1056 3291 1056 2328+Q 1056 1369 1301 889+Q 1547 409 2034 409+Q 2525 409 2770 889+Q 3016 1369 3016 2328+Q 3016 3291 2770 3770+Q 2525 4250 2034 4250+z+M 2034 4750+Q 2819 4750 3233 4129+Q 3647 3509 3647 2328+Q 3647 1150 3233 529+Q 2819 -91 2034 -91+Q 1250 -91 836 529+Q 422 1150 422 2328+Q 422 3509 836 4129+Q 1250 4750 2034 4750+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-13"/>+ </g>+ </g>+ </g>+ <g id="xtick_2">+ <g id="line2d_3">+ <path d="M 180.09 350.532+L 180.09 138.672+" clip-path="url(#p4105837502)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_4">+ <g>+ <use xlink:href="#m328dabad00" x="180.09" y="350.532" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_2">+ <!-- 500 -->+ <g style="fill: #444444" transform="translate(167.683125 369.408953) scale(0.13 -0.13)">+ <defs>+ <path id="DejaVuSans-18" d="M 691 4666+L 3169 4666+L 3169 4134+L 1269 4134+L 1269 2991+Q 1406 3038 1543 3061+Q 1681 3084 1819 3084+Q 2600 3084 3056 2656+Q 3513 2228 3513 1497+Q 3513 744 3044 326+Q 2575 -91 1722 -91+Q 1428 -91 1123 -41+Q 819 9 494 109+L 494 744+Q 775 591 1075 516+Q 1375 441 1709 441+Q 2250 441 2565 725+Q 2881 1009 2881 1497+Q 2881 1984 2565 2268+Q 2250 2553 1709 2553+Q 1456 2553 1204 2497+Q 953 2441 691 2322+L 691 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="xtick_3">+ <g id="line2d_5">+ <path d="M 275.724 350.532+L 275.724 138.672+" clip-path="url(#p4105837502)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_6">+ <g>+ <use xlink:href="#m328dabad00" x="275.724" y="350.532" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_3">+ <!-- 1000 -->+ <g style="fill: #444444" transform="translate(259.1815 369.408953) scale(0.13 -0.13)">+ <defs>+ <path id="DejaVuSans-14" d="M 794 531+L 1825 531+L 1825 4091+L 703 3866+L 703 4441+L 1819 4666+L 2450 4666+L 2450 531+L 3481 531+L 3481 0+L 794 0+L 794 531+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-14"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(190.875 0)"/>+ </g>+ </g>+ </g>+ <g id="xtick_4">+ <g id="line2d_7">+ <path d="M 371.358 350.532+L 371.358 138.672+" clip-path="url(#p4105837502)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_8">+ <g>+ <use xlink:href="#m328dabad00" x="371.358" y="350.532" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_4">+ <!-- 1500 -->+ <g style="fill: #444444" transform="translate(354.8155 369.408953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-14"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(190.875 0)"/>+ </g>+ </g>+ </g>+ <g id="xtick_5">+ <g id="line2d_9">+ <path d="M 466.992 350.532+L 466.992 138.672+" clip-path="url(#p4105837502)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_10">+ <g>+ <use xlink:href="#m328dabad00" x="466.992" y="350.532" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_5">+ <!-- 2000 -->+ <g style="fill: #444444" transform="translate(450.4495 369.408953) scale(0.13 -0.13)">+ <defs>+ <path id="DejaVuSans-15" d="M 1228 531+L 3431 531+L 3431 0+L 469 0+L 469 531+Q 828 903 1448 1529+Q 2069 2156 2228 2338+Q 2531 2678 2651 2914+Q 2772 3150 2772 3378+Q 2772 3750 2511 3984+Q 2250 4219 1831 4219+Q 1534 4219 1204 4116+Q 875 4013 500 3803+L 500 4441+Q 881 4594 1212 4672+Q 1544 4750 1819 4750+Q 2544 4750 2975 4387+Q 3406 4025 3406 3419+Q 3406 3131 3298 2873+Q 3191 2616 2906 2266+Q 2828 2175 2409 1742+Q 1991 1309 1228 531+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-15"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(190.875 0)"/>+ </g>+ </g>+ </g>+ <g id="text_6">+ <!-- Per-GPU throughput (tokens/s) -->+ <g style="fill: #333333" transform="translate(169.200406 393.169812) scale(0.14 -0.14)">+ <defs>+ <path id="DejaVuSans-33" d="M 1259 4147+L 1259 2394+L 2053 2394+Q 2494 2394 2734 2622+Q 2975 2850 2975 3272+Q 2975 3691 2734 3919+Q 2494 4147 2053 4147+L 1259 4147+z+M 628 4666+L 2053 4666+Q 2838 4666 3239 4311+Q 3641 3956 3641 3272+Q 3641 2581 3239 2228+Q 2838 1875 2053 1875+L 1259 1875+L 1259 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-48" d="M 3597 1894+L 3597 1613+L 953 1613+Q 991 1019 1311 708+Q 1631 397 2203 397+Q 2534 397 2845 478+Q 3156 559 3463 722+L 3463 178+Q 3153 47 2828 -22+Q 2503 -91 2169 -91+Q 1331 -91 842 396+Q 353 884 353 1716+Q 353 2575 817 3079+Q 1281 3584 2069 3584+Q 2775 3584 3186 3129+Q 3597 2675 3597 1894+z+M 3022 2063+Q 3016 2534 2758 2815+Q 2500 3097 2075 3097+Q 1594 3097 1305 2825+Q 1016 2553 972 2059+L 3022 2063+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-55" d="M 2631 2963+Q 2534 3019 2420 3045+Q 2306 3072 2169 3072+Q 1681 3072 1420 2755+Q 1159 2438 1159 1844+L 1159 0+L 581 0+L 581 3500+L 1159 3500+L 1159 2956+Q 1341 3275 1631 3429+Q 1922 3584 2338 3584+Q 2397 3584 2469 3576+Q 2541 3569 2628 3553+L 2631 2963+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-10" d="M 313 2009+L 1997 2009+L 1997 1497+L 313 1497+L 313 2009+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-2a" d="M 3809 666+L 3809 1919+L 2778 1919+L 2778 2438+L 4434 2438+L 4434 434+Q 4069 175 3628 42+Q 3188 -91 2688 -91+Q 1594 -91 976 548+Q 359 1188 359 2328+Q 359 3472 976 4111+Q 1594 4750 2688 4750+Q 3144 4750 3555 4637+Q 3966 4525 4313 4306+L 4313 3634+Q 3963 3931 3569 4081+Q 3175 4231 2741 4231+Q 1884 4231 1454 3753+Q 1025 3275 1025 2328+Q 1025 1384 1454 906+Q 1884 428 2741 428+Q 3075 428 3337 486+Q 3600 544 3809 666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-38" d="M 556 4666+L 1191 4666+L 1191 1831+Q 1191 1081 1462 751+Q 1734 422 2344 422+Q 2950 422 3222 751+Q 3494 1081 3494 1831+L 3494 4666+L 4128 4666+L 4128 1753+Q 4128 841 3676 375+Q 3225 -91 2344 -91+Q 1459 -91 1007 375+Q 556 841 556 1753+L 556 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-3" transform="scale(0.015625)"/>+ <path id="DejaVuSans-57" d="M 1172 4494+L 1172 3500+L 2356 3500+L 2356 3053+L 1172 3053+L 1172 1153+Q 1172 725 1289 603+Q 1406 481 1766 481+L 2356 481+L 2356 0+L 1766 0+Q 1100 0 847 248+Q 594 497 594 1153+L 594 3053+L 172 3053+L 172 3500+L 594 3500+L 594 4494+L 1172 4494+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-4b" d="M 3513 2113+L 3513 0+L 2938 0+L 2938 2094+Q 2938 2591 2744 2837+Q 2550 3084 2163 3084+Q 1697 3084 1428 2787+Q 1159 2491 1159 1978+L 1159 0+L 581 0+L 581 4863+L 1159 4863+L 1159 2956+Q 1366 3272 1645 3428+Q 1925 3584 2291 3584+Q 2894 3584 3203 3211+Q 3513 2838 3513 2113+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-52" d="M 1959 3097+Q 1497 3097 1228 2736+Q 959 2375 959 1747+Q 959 1119 1226 758+Q 1494 397 1959 397+Q 2419 397 2687 759+Q 2956 1122 2956 1747+Q 2956 2369 2687 2733+Q 2419 3097 1959 3097+z+M 1959 3584+Q 2709 3584 3137 3096+Q 3566 2609 3566 1747+Q 3566 888 3137 398+Q 2709 -91 1959 -91+Q 1206 -91 779 398+Q 353 888 353 1747+Q 353 2609 779 3096+Q 1206 3584 1959 3584+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-58" d="M 544 1381+L 544 3500+L 1119 3500+L 1119 1403+Q 1119 906 1312 657+Q 1506 409 1894 409+Q 2359 409 2629 706+Q 2900 1003 2900 1516+L 2900 3500+L 3475 3500+L 3475 0+L 2900 0+L 2900 538+Q 2691 219 2414 64+Q 2138 -91 1772 -91+Q 1169 -91 856 284+Q 544 659 544 1381+z+M 1991 3584+L 1991 3584+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-4a" d="M 2906 1791+Q 2906 2416 2648 2759+Q 2391 3103 1925 3103+Q 1463 3103 1205 2759+Q 947 2416 947 1791+Q 947 1169 1205 825+Q 1463 481 1925 481+Q 2391 481 2648 825+Q 2906 1169 2906 1791+z+M 3481 434+Q 3481 -459 3084 -895+Q 2688 -1331 1869 -1331+Q 1566 -1331 1297 -1286+Q 1028 -1241 775 -1147+L 775 -588+Q 1028 -725 1275 -790+Q 1522 -856 1778 -856+Q 2344 -856 2625 -561+Q 2906 -266 2906 331+L 2906 616+Q 2728 306 2450 153+Q 2172 0 1784 0+Q 1141 0 747 490+Q 353 981 353 1791+Q 353 2603 747 3093+Q 1141 3584 1784 3584+Q 2172 3584 2450 3431+Q 2728 3278 2906 2969+L 2906 3500+L 3481 3500+L 3481 434+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-53" d="M 1159 525+L 1159 -1331+L 581 -1331+L 581 3500+L 1159 3500+L 1159 2969+Q 1341 3281 1617 3432+Q 1894 3584 2278 3584+Q 2916 3584 3314 3078+Q 3713 2572 3713 1747+Q 3713 922 3314 415+Q 2916 -91 2278 -91+Q 1894 -91 1617 61+Q 1341 213 1159 525+z+M 3116 1747+Q 3116 2381 2855 2742+Q 2594 3103 2138 3103+Q 1681 3103 1420 2742+Q 1159 2381 1159 1747+Q 1159 1113 1420 752+Q 1681 391 2138 391+Q 2594 391 2855 752+Q 3116 1113 3116 1747+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-b" d="M 1984 4856+Q 1566 4138 1362 3434+Q 1159 2731 1159 2009+Q 1159 1288 1364 580+Q 1569 -128 1984 -844+L 1484 -844+Q 1016 -109 783 600+Q 550 1309 550 2009+Q 550 2706 781 3412+Q 1013 4119 1484 4856+L 1984 4856+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-4e" d="M 581 4863+L 1159 4863+L 1159 1991+L 2875 3500+L 3609 3500+L 1753 1863+L 3688 0+L 2938 0+L 1159 1709+L 1159 0+L 581 0+L 581 4863+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-51" d="M 3513 2113+L 3513 0+L 2938 0+L 2938 2094+Q 2938 2591 2744 2837+Q 2550 3084 2163 3084+Q 1697 3084 1428 2787+Q 1159 2491 1159 1978+L 1159 0+L 581 0+L 581 3500+L 1159 3500+L 1159 2956+Q 1366 3272 1645 3428+Q 1925 3584 2291 3584+Q 2894 3584 3203 3211+Q 3513 2838 3513 2113+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-56" d="M 2834 3397+L 2834 2853+Q 2591 2978 2328 3040+Q 2066 3103 1784 3103+Q 1356 3103 1142 2972+Q 928 2841 928 2578+Q 928 2378 1081 2264+Q 1234 2150 1697 2047+L 1894 2003+Q 2506 1872 2764 1633+Q 3022 1394 3022 966+Q 3022 478 2636 193+Q 2250 -91 1575 -91+Q 1294 -91 989 -36+Q 684 19 347 128+L 347 722+Q 666 556 975 473+Q 1284 391 1588 391+Q 1994 391 2212 530+Q 2431 669 2431 922+Q 2431 1156 2273 1281+Q 2116 1406 1581 1522+L 1381 1569+Q 847 1681 609 1914+Q 372 2147 372 2553+Q 372 3047 722 3315+Q 1072 3584 1716 3584+Q 2034 3584 2315 3537+Q 2597 3491 2834 3397+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-12" d="M 1625 4666+L 2156 4666+L 531 -594+L 0 -594+L 1625 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-c" d="M 513 4856+L 1013 4856+Q 1481 4119 1714 3412+Q 1947 2706 1947 2009+Q 1947 1309 1714 600+Q 1481 -109 1013 -844+L 513 -844+Q 928 -128 1133 580+Q 1338 1288 1338 2009+Q 1338 2731 1133 3434+Q 928 4138 513 4856+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-33"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(56.734375 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(118.265625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(152.984375 0)"/>+ <use xlink:href="#DejaVuSans-2a" transform="translate(192.71875 0)"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(270.203125 0)"/>+ <use xlink:href="#DejaVuSans-38" transform="translate(330.5 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(403.6875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(435.46875 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(474.671875 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(538.046875 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(576.953125 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(638.140625 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(701.515625 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(765 0)"/>+ <use xlink:href="#DejaVuSans-53" transform="translate(828.375 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(891.859375 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(955.234375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(994.4375 0)"/>+ <use xlink:href="#DejaVuSans-b" transform="translate(1026.21875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(1065.234375 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(1104.4375 0)"/>+ <use xlink:href="#DejaVuSans-4e" transform="translate(1165.625 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(1219.96875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(1281.5 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1344.875 0)"/>+ <use xlink:href="#DejaVuSans-12" transform="translate(1396.96875 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1430.65625 0)"/>+ <use xlink:href="#DejaVuSans-c" transform="translate(1482.75 0)"/>+ </g>+ </g>+ </g>+ <g id="matplotlib.axis_2">+ <g id="ytick_1">+ <g id="line2d_11">+ <path d="M 84.456 350.532+L 466.992 350.532+" clip-path="url(#p4105837502)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_12">+ <defs>+ <path id="m48df5c1fc4" d="M 0 0+L -4 0+" style="stroke: #444444; stroke-width: 0.8"/>+ </defs>+ <g>+ <use xlink:href="#m48df5c1fc4" x="84.456" y="350.532" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_7">+ <!-- 0 -->+ <g style="fill: #444444" transform="translate(67.18475 355.470477) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-13"/>+ </g>+ </g>+ </g>+ <g id="ytick_2">+ <g id="line2d_13">+ <path d="M 84.456 288.220235+L 466.992 288.220235+" clip-path="url(#p4105837502)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_14">+ <g>+ <use xlink:href="#m48df5c1fc4" x="84.456" y="288.220235" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_8">+ <!-- 100 -->+ <g style="fill: #444444" transform="translate(50.64225 293.158712) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-14"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="ytick_3">+ <g id="line2d_15">+ <path d="M 84.456 225.908471+L 466.992 225.908471+" clip-path="url(#p4105837502)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_16">+ <g>+ <use xlink:href="#m48df5c1fc4" x="84.456" y="225.908471" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_9">+ <!-- 200 -->+ <g style="fill: #444444" transform="translate(50.64225 230.846947) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-15"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="ytick_4">+ <g id="line2d_17">+ <path d="M 84.456 163.596706+L 466.992 163.596706+" clip-path="url(#p4105837502)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_18">+ <g>+ <use xlink:href="#m48df5c1fc4" x="84.456" y="163.596706" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_10">+ <!-- 300 -->+ <g style="fill: #444444" transform="translate(50.64225 168.535182) scale(0.13 -0.13)">+ <defs>+ <path id="DejaVuSans-16" d="M 2597 2516+Q 3050 2419 3304 2112+Q 3559 1806 3559 1356+Q 3559 666 3084 287+Q 2609 -91 1734 -91+Q 1441 -91 1130 -33+Q 819 25 488 141+L 488 750+Q 750 597 1062 519+Q 1375 441 1716 441+Q 2309 441 2620 675+Q 2931 909 2931 1356+Q 2931 1769 2642 2001+Q 2353 2234 1838 2234+L 1294 2234+L 1294 2753+L 1863 2753+Q 2328 2753 2575 2939+Q 2822 3125 2822 3475+Q 2822 3834 2567 4026+Q 2313 4219 1838 4219+Q 1578 4219 1281 4162+Q 984 4106 628 3988+L 628 4550+Q 988 4650 1302 4700+Q 1616 4750 1894 4750+Q 2613 4750 3031 4423+Q 3450 4097 3450 3541+Q 3450 3153 3228 2886+Q 3006 2619 2597 2516+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="text_11">+ <!-- Per-request throughput (tokens/s) -->+ <g style="fill: #333333" transform="translate(36.278969 362.710594) rotate(-90) scale(0.14 -0.14)">+ <defs>+ <path id="DejaVuSans-54" d="M 947 1747+Q 947 1113 1208 752+Q 1469 391 1925 391+Q 2381 391 2643 752+Q 2906 1113 2906 1747+Q 2906 2381 2643 2742+Q 2381 3103 1925 3103+Q 1469 3103 1208 2742+Q 947 2381 947 1747+z+M 2906 525+Q 2725 213 2448 61+Q 2172 -91 1784 -91+Q 1150 -91 751 415+Q 353 922 353 1747+Q 353 2572 751 3078+Q 1150 3584 1784 3584+Q 2172 3584 2448 3432+Q 2725 3281 2906 2969+L 2906 3500+L 3481 3500+L 3481 -1331+L 2906 -1331+L 2906 525+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-33"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(56.734375 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(118.265625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(152.984375 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(189.0625 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(227.96875 0)"/>+ <use xlink:href="#DejaVuSans-54" transform="translate(289.5 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(352.984375 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(416.359375 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(477.890625 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(529.984375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(569.1875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(600.96875 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(640.171875 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(703.546875 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(742.453125 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(803.640625 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(867.015625 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(930.5 0)"/>+ <use xlink:href="#DejaVuSans-53" transform="translate(993.875 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(1057.359375 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(1120.734375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(1159.9375 0)"/>+ <use xlink:href="#DejaVuSans-b" transform="translate(1191.71875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(1230.734375 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(1269.9375 0)"/>+ <use xlink:href="#DejaVuSans-4e" transform="translate(1331.125 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(1385.46875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(1447 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1510.375 0)"/>+ <use xlink:href="#DejaVuSans-12" transform="translate(1562.46875 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1596.15625 0)"/>+ <use xlink:href="#DejaVuSans-c" transform="translate(1648.25 0)"/>+ </g>+ </g>+ </g>+ <g id="patch_3">+ <path d="M 84.456 350.532+L 84.456 138.672+" style="fill: none; stroke: #888888; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>+ </g>+ <g id="patch_4">+ <path d="M 84.456 350.532+L 466.992 350.532+" style="fill: none; stroke: #888888; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>+ </g>+ <g id="line2d_19">+ <path d="M 130.664902 199.991489+L 197.697101 304.417041+L 210.666564 324.833755+L 199.685262 338.800851+L 202.983846 344.498516+" clip-path="url(#p4105837502)" style="fill: none; stroke-dasharray: 6.66,2.88; stroke-dashoffset: 0; stroke: #7c8594; stroke-width: 1.8"/>+ </g>+ <g id="line2d_20">+ <path d="M 137.181168 178.762635+L 268.099111 275.747363+L 305.14614 305.596386+L 323.71979 326.173265+L 325.774915 338.248019+" clip-path="url(#p4105837502)" style="fill: none; stroke: #4c78b8; stroke-width: 1.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_21">+ <path d="M 90.9165 329.484823+L 100.812547 343.871154+L 102.055668 346.94846+L 102.737878 348.670776+L 103.04121 349.585947+" clip-path="url(#p4105837502)" style="fill: none; stroke: #d9a441; stroke-width: 1.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_22">+ <path d="M 139.038604 172.711436+L 292.708519 265.725721+L 346.947413 297.085059+L 382.029814 320.236895+L 415.062277 333.702979+" clip-path="url(#p4105837502)" style="fill: none; stroke: #bf525b; stroke-width: 1.8; stroke-linecap: square"/>+ </g>+ <g id="text_12">+ <!-- 1 -->+ <g style="fill: #5e8c56" transform="translate(150.20345 146.627446) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-14"/>+ </g>+ </g>+ <g id="text_13">+ <!-- 8 -->+ <g style="fill: #5e8c56" transform="translate(325.783833 248.292663) scale(0.115 -0.115)">+ <defs>+ <path id="DejaVuSans-1b" d="M 2034 2216+Q 1584 2216 1326 1975+Q 1069 1734 1069 1313+Q 1069 891 1326 650+Q 1584 409 2034 409+Q 2484 409 2743 651+Q 3003 894 3003 1313+Q 3003 1734 2745 1975+Q 2488 2216 2034 2216+z+M 1403 2484+Q 997 2584 770 2862+Q 544 3141 544 3541+Q 544 4100 942 4425+Q 1341 4750 2034 4750+Q 2731 4750 3128 4425+Q 3525 4100 3525 3541+Q 3525 3141 3298 2862+Q 3072 2584 2669 2484+Q 3125 2378 3379 2068+Q 3634 1759 3634 1313+Q 3634 634 3220 271+Q 2806 -91 2034 -91+Q 1263 -91 848 271+Q 434 634 434 1313+Q 434 1759 690 2068+Q 947 2378 1403 2484+z+M 1172 3481+Q 1172 3119 1398 2916+Q 1625 2713 2034 2713+Q 2441 2713 2670 2916+Q 2900 3119 2900 3481+Q 2900 3844 2670 4047+Q 2441 4250 2034 4250+Q 1625 4250 1398 4047+Q 1172 3844 1172 3481+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-1b"/>+ </g>+ </g>+ <g id="text_14">+ <!-- 16 -->+ <g style="fill: #5e8c56" transform="translate(392.704985 282.786245) scale(0.115 -0.115)">+ <defs>+ <path id="DejaVuSans-19" d="M 2113 2584+Q 1688 2584 1439 2293+Q 1191 2003 1191 1497+Q 1191 994 1439 701+Q 1688 409 2113 409+Q 2538 409 2786 701+Q 3034 994 3034 1497+Q 3034 2003 2786 2293+Q 2538 2584 2113 2584+z+M 3366 4563+L 3366 3988+Q 3128 4100 2886 4159+Q 2644 4219 2406 4219+Q 1781 4219 1451 3797+Q 1122 3375 1075 2522+Q 1259 2794 1537 2939+Q 1816 3084 2150 3084+Q 2853 3084 3261 2657+Q 3669 2231 3669 1497+Q 3669 778 3244 343+Q 2819 -91 2113 -91+Q 1303 -91 875 529+Q 447 1150 447 2328+Q 447 3434 972 4092+Q 1497 4750 2381 4750+Q 2619 4750 2861 4703+Q 3103 4656 3366 4563+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-14"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ </g>+ </g>+ <g id="text_15">+ <!-- 32 -->+ <g style="fill: #5e8c56" transform="translate(438.230737 309.024281) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ </g>+ </g>+ <g id="text_16">+ <!-- 64 -->+ <g style="fill: #5e8c56" transform="translate(462.793453 325.527812) scale(0.115 -0.115)">+ <defs>+ <path id="DejaVuSans-17" d="M 2419 4116+L 825 1625+L 2419 1625+L 2419 4116+z+M 2253 4666+L 3047 4666+L 3047 1625+L 3713 1625+L 3713 1100+L 3047 1100+L 3047 0+L 2419 0+L 2419 1100+L 313 1100+L 313 1709+L 2253 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(63.625 0)"/>+ </g>+ </g>+ <g id="text_17">+ <!-- Qwen3-4B -->+ <g style="fill: #222222" transform="translate(234.3365 124.672) scale(0.16 -0.16)">+ <defs>+ <path id="DejaVuSans-34" d="M 2522 4238+Q 1834 4238 1429 3725+Q 1025 3213 1025 2328+Q 1025 1447 1429 934+Q 1834 422 2522 422+Q 3209 422 3611 934+Q 4013 1447 4013 2328+Q 4013 3213 3611 3725+Q 3209 4238 2522 4238+z+M 3406 84+L 4238 -825+L 3475 -825+L 2784 -78+Q 2681 -84 2626 -87+Q 2572 -91 2522 -91+Q 1538 -91 948 567+Q 359 1225 359 2328+Q 359 3434 948 4092+Q 1538 4750 2522 4750+Q 3503 4750 4090 4092+Q 4678 3434 4678 2328+Q 4678 1516 4351 937+Q 4025 359 3406 84+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-5a" d="M 269 3500+L 844 3500+L 1563 769+L 2278 3500+L 2956 3500+L 3675 769+L 4391 3500+L 4966 3500+L 4050 0+L 3372 0+L 2619 2869+L 1863 0+L 1184 0+L 269 3500+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-25" d="M 1259 2228+L 1259 519+L 2272 519+Q 2781 519 3026 730+Q 3272 941 3272 1375+Q 3272 1813 3026 2020+Q 2781 2228 2272 2228+L 1259 2228+z+M 1259 4147+L 1259 2741+L 2194 2741+Q 2656 2741 2882 2914+Q 3109 3088 3109 3444+Q 3109 3797 2882 3972+Q 2656 4147 2194 4147+L 1259 4147+z+M 628 4666+L 2241 4666+Q 2963 4666 3353 4366+Q 3744 4066 3744 3513+Q 3744 3084 3544 2831+Q 3344 2578 2956 2516+Q 3422 2416 3680 2098+Q 3938 1781 3938 1306+Q 3938 681 3513 340+Q 3088 0 2303 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-34"/>+ <use xlink:href="#DejaVuSans-5a" transform="translate(78.71875 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(160.5 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(222.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(285.40625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(349.03125 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(385.109375 0)"/>+ <use xlink:href="#DejaVuSans-25" transform="translate(448.734375 0)"/>+ </g>+ </g>+ <g id="line2d_23">+ <path d="M 130.664902 199.991489+" clip-path="url(#p4105837502)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <defs>+ <path id="m2c8cd094e3" d="M 0 3.1+C 0.82213 3.1 1.610698 2.773364 2.192031 2.192031+C 2.773364 1.610698 3.1 0.82213 3.1 0+C 3.1 -0.82213 2.773364 -1.610698 2.192031 -2.192031+C 1.610698 -2.773364 0.82213 -3.1 0 -3.1+C -0.82213 -3.1 -1.610698 -2.773364 -2.192031 -2.192031+C -2.773364 -1.610698 -3.1 -0.82213 -3.1 0+C -3.1 0.82213 -2.773364 1.610698 -2.192031 2.192031+C -1.610698 2.773364 -0.82213 3.1 0 3.1+z+" style="stroke: #ffffff; stroke-width: 0.6"/>+ </defs>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#m2c8cd094e3" x="130.664902" y="199.991489" style="fill: #7c8594; stroke: #ffffff; stroke-width: 0.6"/>+ </g>+ </g>+ <g id="line2d_24">+ <path d="M 197.697101 304.417041+" clip-path="url(#p4105837502)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#m2c8cd094e3" x="197.697101" y="304.417041" style="fill: #7c8594; stroke: #ffffff; stroke-width: 0.6"/>+ </g>+ </g>+ <g id="line2d_25">+ <path d="M 210.666564 324.833755+" clip-path="url(#p4105837502)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#m2c8cd094e3" x="210.666564" y="324.833755" style="fill: #7c8594; stroke: #ffffff; stroke-width: 0.6"/>+ </g>+ </g>+ <g id="line2d_26">+ <path d="M 199.685262 338.800851+" clip-path="url(#p4105837502)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <defs>+ <path id="m3bfe823a7e" d="M 0 3.1+C 0.82213 3.1 1.610698 2.773364 2.192031 2.192031+C 2.773364 1.610698 3.1 0.82213 3.1 0+C 3.1 -0.82213 2.773364 -1.610698 2.192031 -2.192031+C 1.610698 -2.773364 0.82213 -3.1 0 -3.1+C -0.82213 -3.1 -1.610698 -2.773364 -2.192031 -2.192031+C -2.773364 -1.610698 -3.1 -0.82213 -3.1 0+C -3.1 0.82213 -2.773364 1.610698 -2.192031 2.192031+C -1.610698 2.773364 -0.82213 3.1 0 3.1+z+" style="stroke: #7c8594; stroke-width: 1.2"/>+ </defs>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#m3bfe823a7e" x="199.685262" y="338.800851" style="fill: #ffffff; stroke: #7c8594; stroke-width: 1.2"/>+ </g>+ </g>+ <g id="line2d_27">+ <path d="M 202.983846 344.498516+" clip-path="url(#p4105837502)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#m3bfe823a7e" x="202.983846" y="344.498516" style="fill: #ffffff; stroke: #7c8594; stroke-width: 1.2"/>+ </g>+ </g>+ <g id="line2d_28">+ <path d="M 137.181168 178.762635+" clip-path="url(#p4105837502)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <defs>+ <path id="ma7f247753d" d="M 0 -3.1+L -3.1 3.1+L 3.1 3.1+z+" style="stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </defs>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#ma7f247753d" x="137.181168" y="178.762635" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_29">+ <path d="M 268.099111 275.747363+" clip-path="url(#p4105837502)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#ma7f247753d" x="268.099111" y="275.747363" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_30">+ <path d="M 305.14614 305.596386+" clip-path="url(#p4105837502)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#ma7f247753d" x="305.14614" y="305.596386" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_31">+ <path d="M 323.71979 326.173265+" clip-path="url(#p4105837502)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#ma7f247753d" x="323.71979" y="326.173265" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_32">+ <path d="M 325.774915 338.248019+" clip-path="url(#p4105837502)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <defs>+ <path id="m05856f3ac2" d="M 0 -3.1+L -3.1 3.1+L 3.1 3.1+z+" style="stroke: #4c78b8; stroke-width: 1.2; stroke-linejoin: miter"/>+ </defs>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#m05856f3ac2" x="325.774915" y="338.248019" style="fill: #ffffff; stroke: #4c78b8; stroke-width: 1.2; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_33">+ <path d="M 90.9165 329.484823+" clip-path="url(#p4105837502)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <defs>+ <path id="m5d68a1abc3" d="M -0 3.1+L 3.1 -3.1+L -3.1 -3.1+z+" style="stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </defs>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#m5d68a1abc3" x="90.9165" y="329.484823" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_34">+ <path d="M 100.812547 343.871154+" clip-path="url(#p4105837502)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#m5d68a1abc3" x="100.812547" y="343.871154" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_35">+ <path d="M 102.055668 346.94846+" clip-path="url(#p4105837502)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#m5d68a1abc3" x="102.055668" y="346.94846" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_36">+ <path d="M 102.737878 348.670776+" clip-path="url(#p4105837502)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#m5d68a1abc3" x="102.737878" y="348.670776" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_37">+ <path d="M 103.04121 349.585947+" clip-path="url(#p4105837502)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#m5d68a1abc3" x="103.04121" y="349.585947" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_38">+ <path d="M 139.038604 172.711436+" clip-path="url(#p4105837502)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <defs>+ <path id="mef82d167b8" d="M -0 4.384062+L 4.384062 0+L 0 -4.384062+L -4.384062 -0+z+" style="stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </defs>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#mef82d167b8" x="139.038604" y="172.711436" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_39">+ <path d="M 292.708519 265.725721+" clip-path="url(#p4105837502)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#mef82d167b8" x="292.708519" y="265.725721" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_40">+ <path d="M 346.947413 297.085059+" clip-path="url(#p4105837502)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#mef82d167b8" x="346.947413" y="297.085059" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_41">+ <path d="M 382.029814 320.236895+" clip-path="url(#p4105837502)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#mef82d167b8" x="382.029814" y="320.236895" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_42">+ <path d="M 415.062277 333.702979+" clip-path="url(#p4105837502)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#mef82d167b8" x="415.062277" y="333.702979" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_43">+ <path d="M 145.20345 152.627446+L 320.783833 254.292663+L 387.704985 288.786245+L 433.230737 315.024281+L 457.793453 331.527812+" clip-path="url(#p4105837502)" style="fill: none; stroke: #5e8c56; stroke-width: 2.3; stroke-linecap: square"/>+ </g>+ <g id="line2d_44">+ <path d="M 145.20345 152.627446+" clip-path="url(#p4105837502)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <defs>+ <path id="ma29a59a539" d="M -3.5 3.5+L 3.5 3.5+L 3.5 -3.5+L -3.5 -3.5+z+" style="stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </defs>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#ma29a59a539" x="145.20345" y="152.627446" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_45">+ <path d="M 320.783833 254.292663+" clip-path="url(#p4105837502)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#ma29a59a539" x="320.783833" y="254.292663" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_46">+ <path d="M 387.704985 288.786245+" clip-path="url(#p4105837502)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#ma29a59a539" x="387.704985" y="288.786245" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_47">+ <path d="M 433.230737 315.024281+" clip-path="url(#p4105837502)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#ma29a59a539" x="433.230737" y="315.024281" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_48">+ <path d="M 457.793453 331.527812+" clip-path="url(#p4105837502)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p4105837502)">+ <use xlink:href="#ma29a59a539" x="457.793453" y="331.527812" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ </g>+ <g id="axes_2">+ <g id="patch_5">+ <path d="M 561.384 350.532+L 943.92 350.532+L 943.92 138.672+L 561.384 138.672+z+" style="fill: #ffffff"/>+ </g>+ <g id="matplotlib.axis_3">+ <g id="xtick_6">+ <g id="line2d_49">+ <path d="M 561.384 350.532+L 561.384 138.672+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_50">+ <g>+ <use xlink:href="#m328dabad00" x="561.384" y="350.532" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_18">+ <!-- 0 -->+ <g style="fill: #444444" transform="translate(557.248375 369.408953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-13"/>+ </g>+ </g>+ </g>+ <g id="xtick_7">+ <g id="line2d_51">+ <path d="M 637.8912 350.532+L 637.8912 138.672+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_52">+ <g>+ <use xlink:href="#m328dabad00" x="637.8912" y="350.532" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_19">+ <!-- 500 -->+ <g style="fill: #444444" transform="translate(625.484325 369.408953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-18"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="xtick_8">+ <g id="line2d_53">+ <path d="M 714.3984 350.532+L 714.3984 138.672+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_54">+ <g>+ <use xlink:href="#m328dabad00" x="714.3984" y="350.532" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_20">+ <!-- 1000 -->+ <g style="fill: #444444" transform="translate(697.8559 369.408953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-14"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(190.875 0)"/>+ </g>+ </g>+ </g>+ <g id="xtick_9">+ <g id="line2d_55">+ <path d="M 790.9056 350.532+L 790.9056 138.672+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_56">+ <g>+ <use xlink:href="#m328dabad00" x="790.9056" y="350.532" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_21">+ <!-- 1500 -->+ <g style="fill: #444444" transform="translate(774.3631 369.408953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-14"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(190.875 0)"/>+ </g>+ </g>+ </g>+ <g id="xtick_10">+ <g id="line2d_57">+ <path d="M 867.4128 350.532+L 867.4128 138.672+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_58">+ <g>+ <use xlink:href="#m328dabad00" x="867.4128" y="350.532" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_22">+ <!-- 2000 -->+ <g style="fill: #444444" transform="translate(850.8703 369.408953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-15"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(190.875 0)"/>+ </g>+ </g>+ </g>+ <g id="xtick_11">+ <g id="line2d_59">+ <path d="M 943.92 350.532+L 943.92 138.672+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_60">+ <g>+ <use xlink:href="#m328dabad00" x="943.92" y="350.532" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_23">+ <!-- 2500 -->+ <g style="fill: #444444" transform="translate(927.3775 369.408953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-15"/>+ <use xlink:href="#DejaVuSans-18" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(190.875 0)"/>+ </g>+ </g>+ </g>+ <g id="text_24">+ <!-- Per-GPU throughput (tokens/s) -->+ <g style="fill: #333333" transform="translate(646.128406 393.169812) scale(0.14 -0.14)">+ <use xlink:href="#DejaVuSans-33"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(56.734375 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(118.265625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(152.984375 0)"/>+ <use xlink:href="#DejaVuSans-2a" transform="translate(192.71875 0)"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(270.203125 0)"/>+ <use xlink:href="#DejaVuSans-38" transform="translate(330.5 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(403.6875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(435.46875 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(474.671875 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(538.046875 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(576.953125 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(638.140625 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(701.515625 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(765 0)"/>+ <use xlink:href="#DejaVuSans-53" transform="translate(828.375 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(891.859375 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(955.234375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(994.4375 0)"/>+ <use xlink:href="#DejaVuSans-b" transform="translate(1026.21875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(1065.234375 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(1104.4375 0)"/>+ <use xlink:href="#DejaVuSans-4e" transform="translate(1165.625 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(1219.96875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(1281.5 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1344.875 0)"/>+ <use xlink:href="#DejaVuSans-12" transform="translate(1396.96875 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1430.65625 0)"/>+ <use xlink:href="#DejaVuSans-c" transform="translate(1482.75 0)"/>+ </g>+ </g>+ </g>+ <g id="matplotlib.axis_4">+ <g id="ytick_5">+ <g id="line2d_61">+ <path d="M 561.384 350.532+L 943.92 350.532+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_62">+ <g>+ <use xlink:href="#m48df5c1fc4" x="561.384" y="350.532" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_25">+ <!-- 0 -->+ <g style="fill: #444444" transform="translate(544.11275 355.470477) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-13"/>+ </g>+ </g>+ </g>+ <g id="ytick_6">+ <g id="line2d_63">+ <path d="M 561.384 288.220235+L 943.92 288.220235+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_64">+ <g>+ <use xlink:href="#m48df5c1fc4" x="561.384" y="288.220235" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_26">+ <!-- 100 -->+ <g style="fill: #444444" transform="translate(527.57025 293.158712) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-14"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="ytick_7">+ <g id="line2d_65">+ <path d="M 561.384 225.908471+L 943.92 225.908471+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_66">+ <g>+ <use xlink:href="#m48df5c1fc4" x="561.384" y="225.908471" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_27">+ <!-- 200 -->+ <g style="fill: #444444" transform="translate(527.57025 230.846947) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-15"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="ytick_8">+ <g id="line2d_67">+ <path d="M 561.384 163.596706+L 943.92 163.596706+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_68">+ <g>+ <use xlink:href="#m48df5c1fc4" x="561.384" y="163.596706" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_28">+ <!-- 300 -->+ <g style="fill: #444444" transform="translate(527.57025 168.535182) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="text_29">+ <!-- Per-request throughput (tokens/s) -->+ <g style="fill: #333333" transform="translate(513.206969 362.710594) rotate(-90) scale(0.14 -0.14)">+ <use xlink:href="#DejaVuSans-33"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(56.734375 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(118.265625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(152.984375 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(189.0625 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(227.96875 0)"/>+ <use xlink:href="#DejaVuSans-54" transform="translate(289.5 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(352.984375 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(416.359375 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(477.890625 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(529.984375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(569.1875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(600.96875 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(640.171875 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(703.546875 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(742.453125 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(803.640625 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(867.015625 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(930.5 0)"/>+ <use xlink:href="#DejaVuSans-53" transform="translate(993.875 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(1057.359375 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(1120.734375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(1159.9375 0)"/>+ <use xlink:href="#DejaVuSans-b" transform="translate(1191.71875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(1230.734375 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(1269.9375 0)"/>+ <use xlink:href="#DejaVuSans-4e" transform="translate(1331.125 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(1385.46875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(1447 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1510.375 0)"/>+ <use xlink:href="#DejaVuSans-12" transform="translate(1562.46875 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1596.15625 0)"/>+ <use xlink:href="#DejaVuSans-c" transform="translate(1648.25 0)"/>+ </g>+ </g>+ </g>+ <g id="patch_6">+ <path d="M 561.384 350.532+L 561.384 138.672+" style="fill: none; stroke: #888888; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>+ </g>+ <g id="patch_7">+ <path d="M 561.384 350.532+L 943.92 350.532+" style="fill: none; stroke: #888888; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>+ </g>+ <g id="line2d_69">+ <path d="M 596.357808 208.108822+L 657.958924 301.37197+L 679.26885 320.528235+L 676.233297 335.916418+L 681.488587 342.889818+" clip-path="url(#p78789ce146)" style="fill: none; stroke-dasharray: 6.66,2.88; stroke-dashoffset: 0; stroke: #7c8594; stroke-width: 1.8"/>+ </g>+ <g id="line2d_70">+ <path d="M 599.643427 194.728859+L 694.791687 282.6228+L 738.770239 305.384086+L 782.059556 322.449097+L 770.824669 337.205418+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #4c78b8; stroke-width: 1.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_71">+ <path d="M 565.382748 334.247976+L 571.189446 345.540683+L 571.945428 347.843931+L 572.40372 349.129644+L 572.636867 349.815987+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #d9a441; stroke-width: 1.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_72">+ <path d="M 598.70719 198.541479+L 705.281681 277.283025+L 755.687849 301.078257+L 815.45409 318.199349+L 868.223391 331.007996+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #bf525b; stroke-width: 1.8; stroke-linecap: square"/>+ </g>+ <g id="text_30">+ <!-- 1 -->+ <g style="fill: #5e8c56" transform="translate(608.139756 174.490826) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-14"/>+ </g>+ </g>+ <g id="text_31">+ <!-- 8 -->+ <g style="fill: #5e8c56" transform="translate(711.612628 270.605525) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-1b"/>+ </g>+ </g>+ <g id="text_32">+ <!-- 16 -->+ <g style="fill: #5e8c56" transform="translate(775.034769 291.426714) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-14"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ </g>+ </g>+ <g id="text_33">+ <!-- 32 -->+ <g style="fill: #5e8c56" transform="translate(829.74087 311.017525) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ </g>+ </g>+ <g id="text_34">+ <!-- 64 -->+ <g style="fill: #5e8c56" transform="translate(883.648519 324.344652) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(63.625 0)"/>+ </g>+ </g>+ <g id="text_35">+ <!-- Qwen3-30B-A3B -->+ <g style="fill: #222222" transform="translate(687.41325 124.672) scale(0.16 -0.16)">+ <defs>+ <path id="DejaVuSans-24" d="M 2188 4044+L 1331 1722+L 3047 1722+L 2188 4044+z+M 1831 4666+L 2547 4666+L 4325 0+L 3669 0+L 3244 1197+L 1141 1197+L 716 0+L 50 0+L 1831 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-34"/>+ <use xlink:href="#DejaVuSans-5a" transform="translate(78.71875 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(160.5 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(222.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(285.40625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(349.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(385.109375 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(448.734375 0)"/>+ <use xlink:href="#DejaVuSans-25" transform="translate(512.359375 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(580.96875 0)"/>+ <use xlink:href="#DejaVuSans-24" transform="translate(614.84375 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(683.25 0)"/>+ <use xlink:href="#DejaVuSans-25" transform="translate(746.875 0)"/>+ </g>+ </g>+ <g id="line2d_73">+ <path d="M 596.357808 208.108822+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#m2c8cd094e3" x="596.357808" y="208.108822" style="fill: #7c8594; stroke: #ffffff; stroke-width: 0.6"/>+ </g>+ </g>+ <g id="line2d_74">+ <path d="M 657.958924 301.37197+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#m2c8cd094e3" x="657.958924" y="301.37197" style="fill: #7c8594; stroke: #ffffff; stroke-width: 0.6"/>+ </g>+ </g>+ <g id="line2d_75">+ <path d="M 679.26885 320.528235+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#m2c8cd094e3" x="679.26885" y="320.528235" style="fill: #7c8594; stroke: #ffffff; stroke-width: 0.6"/>+ </g>+ </g>+ <g id="line2d_76">+ <path d="M 676.233297 335.916418+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#m3bfe823a7e" x="676.233297" y="335.916418" style="fill: #ffffff; stroke: #7c8594; stroke-width: 1.2"/>+ </g>+ </g>+ <g id="line2d_77">+ <path d="M 681.488587 342.889818+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#m3bfe823a7e" x="681.488587" y="342.889818" style="fill: #ffffff; stroke: #7c8594; stroke-width: 1.2"/>+ </g>+ </g>+ <g id="line2d_78">+ <path d="M 599.643427 194.728859+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#ma7f247753d" x="599.643427" y="194.728859" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_79">+ <path d="M 694.791687 282.6228+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#ma7f247753d" x="694.791687" y="282.6228" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_80">+ <path d="M 738.770239 305.384086+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#ma7f247753d" x="738.770239" y="305.384086" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_81">+ <path d="M 782.059556 322.449097+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#ma7f247753d" x="782.059556" y="322.449097" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_82">+ <path d="M 770.824669 337.205418+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#m05856f3ac2" x="770.824669" y="337.205418" style="fill: #ffffff; stroke: #4c78b8; stroke-width: 1.2; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_83">+ <path d="M 565.382748 334.247976+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#m5d68a1abc3" x="565.382748" y="334.247976" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_84">+ <path d="M 571.189446 345.540683+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#m5d68a1abc3" x="571.189446" y="345.540683" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_85">+ <path d="M 571.945428 347.843931+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#m5d68a1abc3" x="571.945428" y="347.843931" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_86">+ <path d="M 572.40372 349.129644+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#m5d68a1abc3" x="572.40372" y="349.129644" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_87">+ <path d="M 572.636867 349.815987+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#m5d68a1abc3" x="572.636867" y="349.815987" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_88">+ <path d="M 598.70719 198.541479+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#mef82d167b8" x="598.70719" y="198.541479" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_89">+ <path d="M 705.281681 277.283025+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#mef82d167b8" x="705.281681" y="277.283025" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_90">+ <path d="M 755.687849 301.078257+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#mef82d167b8" x="755.687849" y="301.078257" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_91">+ <path d="M 815.45409 318.199349+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#mef82d167b8" x="815.45409" y="318.199349" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_92">+ <path d="M 868.223391 331.007996+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#mef82d167b8" x="868.223391" y="331.007996" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_93">+ <path d="M 603.139756 180.490826+L 706.612628 276.605525+L 770.034769 297.426714+L 824.74087 317.017525+L 878.648519 330.344652+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #5e8c56; stroke-width: 2.3; stroke-linecap: square"/>+ </g>+ <g id="line2d_94">+ <path d="M 603.139756 180.490826+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#ma29a59a539" x="603.139756" y="180.490826" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_95">+ <path d="M 706.612628 276.605525+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#ma29a59a539" x="706.612628" y="276.605525" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_96">+ <path d="M 770.034769 297.426714+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#ma29a59a539" x="770.034769" y="297.426714" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_97">+ <path d="M 824.74087 317.017525+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#ma29a59a539" x="824.74087" y="317.017525" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_98">+ <path d="M 878.648519 330.344652+" clip-path="url(#p78789ce146)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p78789ce146)">+ <use xlink:href="#ma29a59a539" x="878.648519" y="330.344652" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ </g>+ <g id="axes_3">+ <g id="patch_8">+ <path d="M 84.456 704.916+L 466.992 704.916+L 466.992 493.056+L 84.456 493.056+z+" style="fill: #ffffff"/>+ </g>+ <g id="matplotlib.axis_5">+ <g id="xtick_12">+ <g id="line2d_99">+ <path d="M 84.456 704.916+L 84.456 493.056+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_100">+ <g>+ <use xlink:href="#m328dabad00" x="84.456" y="704.916" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_36">+ <!-- 0 -->+ <g style="fill: #444444" transform="translate(80.320375 723.792953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-13"/>+ </g>+ </g>+ </g>+ <g id="xtick_13">+ <g id="line2d_101">+ <path d="M 211.968 704.916+L 211.968 493.056+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_102">+ <g>+ <use xlink:href="#m328dabad00" x="211.968" y="704.916" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_37">+ <!-- 400 -->+ <g style="fill: #444444" transform="translate(199.561125 723.792953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-17"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="xtick_14">+ <g id="line2d_103">+ <path d="M 339.48 704.916+L 339.48 493.056+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_104">+ <g>+ <use xlink:href="#m328dabad00" x="339.48" y="704.916" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_38">+ <!-- 800 -->+ <g style="fill: #444444" transform="translate(327.073125 723.792953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="xtick_15">+ <g id="line2d_105">+ <path d="M 466.992 704.916+L 466.992 493.056+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_106">+ <g>+ <use xlink:href="#m328dabad00" x="466.992" y="704.916" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_39">+ <!-- 1200 -->+ <g style="fill: #444444" transform="translate(450.4495 723.792953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-14"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(190.875 0)"/>+ </g>+ </g>+ </g>+ <g id="text_40">+ <!-- Per-GPU throughput (tokens/s) -->+ <g style="fill: #333333" transform="translate(169.200406 747.553812) scale(0.14 -0.14)">+ <use xlink:href="#DejaVuSans-33"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(56.734375 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(118.265625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(152.984375 0)"/>+ <use xlink:href="#DejaVuSans-2a" transform="translate(192.71875 0)"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(270.203125 0)"/>+ <use xlink:href="#DejaVuSans-38" transform="translate(330.5 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(403.6875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(435.46875 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(474.671875 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(538.046875 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(576.953125 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(638.140625 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(701.515625 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(765 0)"/>+ <use xlink:href="#DejaVuSans-53" transform="translate(828.375 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(891.859375 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(955.234375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(994.4375 0)"/>+ <use xlink:href="#DejaVuSans-b" transform="translate(1026.21875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(1065.234375 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(1104.4375 0)"/>+ <use xlink:href="#DejaVuSans-4e" transform="translate(1165.625 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(1219.96875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(1281.5 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1344.875 0)"/>+ <use xlink:href="#DejaVuSans-12" transform="translate(1396.96875 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1430.65625 0)"/>+ <use xlink:href="#DejaVuSans-c" transform="translate(1482.75 0)"/>+ </g>+ </g>+ </g>+ <g id="matplotlib.axis_6">+ <g id="ytick_9">+ <g id="line2d_107">+ <path d="M 84.456 704.916+L 466.992 704.916+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_108">+ <g>+ <use xlink:href="#m48df5c1fc4" x="84.456" y="704.916" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_41">+ <!-- 0 -->+ <g style="fill: #444444" transform="translate(67.18475 709.854477) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-13"/>+ </g>+ </g>+ </g>+ <g id="ytick_10">+ <g id="line2d_109">+ <path d="M 84.456 642.604235+L 466.992 642.604235+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_110">+ <g>+ <use xlink:href="#m48df5c1fc4" x="84.456" y="642.604235" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_42">+ <!-- 100 -->+ <g style="fill: #444444" transform="translate(50.64225 647.542712) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-14"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="ytick_11">+ <g id="line2d_111">+ <path d="M 84.456 580.292471+L 466.992 580.292471+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_112">+ <g>+ <use xlink:href="#m48df5c1fc4" x="84.456" y="580.292471" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_43">+ <!-- 200 -->+ <g style="fill: #444444" transform="translate(50.64225 585.230947) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-15"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="ytick_12">+ <g id="line2d_113">+ <path d="M 84.456 517.980706+L 466.992 517.980706+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_114">+ <g>+ <use xlink:href="#m48df5c1fc4" x="84.456" y="517.980706" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_44">+ <!-- 300 -->+ <g style="fill: #444444" transform="translate(50.64225 522.919182) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="text_45">+ <!-- Per-request throughput (tokens/s) -->+ <g style="fill: #333333" transform="translate(36.278969 717.094594) rotate(-90) scale(0.14 -0.14)">+ <use xlink:href="#DejaVuSans-33"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(56.734375 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(118.265625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(152.984375 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(189.0625 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(227.96875 0)"/>+ <use xlink:href="#DejaVuSans-54" transform="translate(289.5 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(352.984375 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(416.359375 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(477.890625 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(529.984375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(569.1875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(600.96875 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(640.171875 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(703.546875 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(742.453125 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(803.640625 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(867.015625 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(930.5 0)"/>+ <use xlink:href="#DejaVuSans-53" transform="translate(993.875 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(1057.359375 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(1120.734375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(1159.9375 0)"/>+ <use xlink:href="#DejaVuSans-b" transform="translate(1191.71875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(1230.734375 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(1269.9375 0)"/>+ <use xlink:href="#DejaVuSans-4e" transform="translate(1331.125 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(1385.46875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(1447 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1510.375 0)"/>+ <use xlink:href="#DejaVuSans-12" transform="translate(1562.46875 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1596.15625 0)"/>+ <use xlink:href="#DejaVuSans-c" transform="translate(1648.25 0)"/>+ </g>+ </g>+ </g>+ <g id="patch_9">+ <path d="M 84.456 704.916+L 84.456 493.056+" style="fill: none; stroke: #888888; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>+ </g>+ <g id="patch_10">+ <path d="M 84.456 704.916+L 466.992 704.916+" style="fill: none; stroke: #888888; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>+ </g>+ <g id="line2d_115">+ <path d="M 124.047634 550.136867+L 179.915278 658.267558+L 190.884996 678.911473+L 188.60941 692.191741+L 192.120426 698.339403+" clip-path="url(#p9992f1a396)" style="fill: none; stroke-dasharray: 6.66,2.88; stroke-dashoffset: 0; stroke: #7c8594; stroke-width: 1.8"/>+ </g>+ <g id="line2d_116">+ <path d="M 130.958107 523.121135+L 242.167871 627.846352+L 270.319204 659.502767+L 291.277147 679.648985+L 277.034372 693.152501+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #4c78b8; stroke-width: 1.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_117">+ <path d="M 87.540537 692.85734+L 97.283468 698.647554+L 98.446119 701.497698+L 99.307513 703.101614+L 99.676173 703.986288+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #d9a441; stroke-width: 1.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_118">+ <path d="M 131.547378 520.817444+L 262.242741 618.03629+L 304.4593 651.161083+L 339.216962 673.792253+L 357.891492 688.213407+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #bf525b; stroke-width: 1.8; stroke-linecap: square"/>+ </g>+ <g id="text_46">+ <!-- 1 -->+ <g style="fill: #5e8c56" transform="translate(137.873545 509.632941) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-14"/>+ </g>+ </g>+ <g id="text_47">+ <!-- 8 -->+ <g style="fill: #5e8c56" transform="translate(294.29809 598.815052) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-1b"/>+ </g>+ </g>+ <g id="text_48">+ <!-- 16 -->+ <g style="fill: #5e8c56" transform="translate(345.065836 636.461094) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-14"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ </g>+ </g>+ <g id="text_49">+ <!-- 32 -->+ <g style="fill: #5e8c56" transform="translate(391.181687 662.054646) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ </g>+ </g>+ <g id="text_50">+ <!-- 64 -->+ <g style="fill: #5e8c56" transform="translate(412.061033 679.209924) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(63.625 0)"/>+ </g>+ </g>+ <g id="text_51">+ <!-- Qwen3-4B -->+ <g style="fill: #222222" transform="translate(234.3365 479.056) scale(0.16 -0.16)">+ <use xlink:href="#DejaVuSans-34"/>+ <use xlink:href="#DejaVuSans-5a" transform="translate(78.71875 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(160.5 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(222.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(285.40625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(349.03125 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(385.109375 0)"/>+ <use xlink:href="#DejaVuSans-25" transform="translate(448.734375 0)"/>+ </g>+ </g>+ <g id="line2d_119">+ <path d="M 124.047634 550.136867+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#m2c8cd094e3" x="124.047634" y="550.136867" style="fill: #7c8594; stroke: #ffffff; stroke-width: 0.6"/>+ </g>+ </g>+ <g id="line2d_120">+ <path d="M 179.915278 658.267558+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#m2c8cd094e3" x="179.915278" y="658.267558" style="fill: #7c8594; stroke: #ffffff; stroke-width: 0.6"/>+ </g>+ </g>+ <g id="line2d_121">+ <path d="M 190.884996 678.911473+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#m2c8cd094e3" x="190.884996" y="678.911473" style="fill: #7c8594; stroke: #ffffff; stroke-width: 0.6"/>+ </g>+ </g>+ <g id="line2d_122">+ <path d="M 188.60941 692.191741+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#m3bfe823a7e" x="188.60941" y="692.191741" style="fill: #ffffff; stroke: #7c8594; stroke-width: 1.2"/>+ </g>+ </g>+ <g id="line2d_123">+ <path d="M 192.120426 698.339403+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#m3bfe823a7e" x="192.120426" y="698.339403" style="fill: #ffffff; stroke: #7c8594; stroke-width: 1.2"/>+ </g>+ </g>+ <g id="line2d_124">+ <path d="M 130.958107 523.121135+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#ma7f247753d" x="130.958107" y="523.121135" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_125">+ <path d="M 242.167871 627.846352+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#ma7f247753d" x="242.167871" y="627.846352" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_126">+ <path d="M 270.319204 659.502767+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#ma7f247753d" x="270.319204" y="659.502767" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_127">+ <path d="M 291.277147 679.648985+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#ma7f247753d" x="291.277147" y="679.648985" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_128">+ <path d="M 277.034372 693.152501+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#m05856f3ac2" x="277.034372" y="693.152501" style="fill: #ffffff; stroke: #4c78b8; stroke-width: 1.2; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_129">+ <path d="M 87.540537 692.85734+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#m5d68a1abc3" x="87.540537" y="692.85734" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_130">+ <path d="M 97.283468 698.647554+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#m5d68a1abc3" x="97.283468" y="698.647554" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_131">+ <path d="M 98.446119 701.497698+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#m5d68a1abc3" x="98.446119" y="701.497698" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_132">+ <path d="M 99.307513 703.101614+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#m5d68a1abc3" x="99.307513" y="703.101614" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_133">+ <path d="M 99.676173 703.986288+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#m5d68a1abc3" x="99.676173" y="703.986288" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_134">+ <path d="M 131.547378 520.817444+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#mef82d167b8" x="131.547378" y="520.817444" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_135">+ <path d="M 262.242741 618.03629+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#mef82d167b8" x="262.242741" y="618.03629" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_136">+ <path d="M 304.4593 651.161083+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#mef82d167b8" x="304.4593" y="651.161083" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_137">+ <path d="M 339.216962 673.792253+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#mef82d167b8" x="339.216962" y="673.792253" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_138">+ <path d="M 357.891492 688.213407+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#mef82d167b8" x="357.891492" y="688.213407" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_139">+ <path d="M 132.873545 515.632941+L 289.29809 604.815052+L 340.065836 642.461094+L 386.181687 668.054646+L 407.061033 685.209924+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #5e8c56; stroke-width: 2.3; stroke-linecap: square"/>+ </g>+ <g id="line2d_140">+ <path d="M 132.873545 515.632941+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#ma29a59a539" x="132.873545" y="515.632941" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_141">+ <path d="M 289.29809 604.815052+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#ma29a59a539" x="289.29809" y="604.815052" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_142">+ <path d="M 340.065836 642.461094+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#ma29a59a539" x="340.065836" y="642.461094" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_143">+ <path d="M 386.181687 668.054646+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#ma29a59a539" x="386.181687" y="668.054646" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_144">+ <path d="M 407.061033 685.209924+" clip-path="url(#p9992f1a396)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p9992f1a396)">+ <use xlink:href="#ma29a59a539" x="407.061033" y="685.209924" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ </g>+ <g id="axes_4">+ <g id="patch_11">+ <path d="M 561.384 704.916+L 943.92 704.916+L 943.92 493.056+L 561.384 493.056+z+" style="fill: #ffffff"/>+ </g>+ <g id="matplotlib.axis_7">+ <g id="xtick_16">+ <g id="line2d_145">+ <path d="M 561.384 704.916+L 561.384 493.056+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_146">+ <g>+ <use xlink:href="#m328dabad00" x="561.384" y="704.916" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_52">+ <!-- 0 -->+ <g style="fill: #444444" transform="translate(557.248375 723.792953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-13"/>+ </g>+ </g>+ </g>+ <g id="xtick_17">+ <g id="line2d_147">+ <path d="M 688.896 704.916+L 688.896 493.056+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_148">+ <g>+ <use xlink:href="#m328dabad00" x="688.896" y="704.916" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_53">+ <!-- 400 -->+ <g style="fill: #444444" transform="translate(676.489125 723.792953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-17"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="xtick_18">+ <g id="line2d_149">+ <path d="M 816.408 704.916+L 816.408 493.056+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_150">+ <g>+ <use xlink:href="#m328dabad00" x="816.408" y="704.916" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_54">+ <!-- 800 -->+ <g style="fill: #444444" transform="translate(804.001125 723.792953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-1b"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="xtick_19">+ <g id="line2d_151">+ <path d="M 943.92 704.916+L 943.92 493.056+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_152">+ <g>+ <use xlink:href="#m328dabad00" x="943.92" y="704.916" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_55">+ <!-- 1200 -->+ <g style="fill: #444444" transform="translate(927.3775 723.792953) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-14"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(190.875 0)"/>+ </g>+ </g>+ </g>+ <g id="text_56">+ <!-- Per-GPU throughput (tokens/s) -->+ <g style="fill: #333333" transform="translate(646.128406 747.553812) scale(0.14 -0.14)">+ <use xlink:href="#DejaVuSans-33"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(56.734375 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(118.265625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(152.984375 0)"/>+ <use xlink:href="#DejaVuSans-2a" transform="translate(192.71875 0)"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(270.203125 0)"/>+ <use xlink:href="#DejaVuSans-38" transform="translate(330.5 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(403.6875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(435.46875 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(474.671875 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(538.046875 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(576.953125 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(638.140625 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(701.515625 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(765 0)"/>+ <use xlink:href="#DejaVuSans-53" transform="translate(828.375 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(891.859375 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(955.234375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(994.4375 0)"/>+ <use xlink:href="#DejaVuSans-b" transform="translate(1026.21875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(1065.234375 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(1104.4375 0)"/>+ <use xlink:href="#DejaVuSans-4e" transform="translate(1165.625 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(1219.96875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(1281.5 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1344.875 0)"/>+ <use xlink:href="#DejaVuSans-12" transform="translate(1396.96875 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1430.65625 0)"/>+ <use xlink:href="#DejaVuSans-c" transform="translate(1482.75 0)"/>+ </g>+ </g>+ </g>+ <g id="matplotlib.axis_8">+ <g id="ytick_13">+ <g id="line2d_153">+ <path d="M 561.384 704.916+L 943.92 704.916+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_154">+ <g>+ <use xlink:href="#m48df5c1fc4" x="561.384" y="704.916" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_57">+ <!-- 0 -->+ <g style="fill: #444444" transform="translate(544.11275 709.854477) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-13"/>+ </g>+ </g>+ </g>+ <g id="ytick_14">+ <g id="line2d_155">+ <path d="M 561.384 642.604235+L 943.92 642.604235+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_156">+ <g>+ <use xlink:href="#m48df5c1fc4" x="561.384" y="642.604235" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_58">+ <!-- 100 -->+ <g style="fill: #444444" transform="translate(527.57025 647.542712) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-14"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="ytick_15">+ <g id="line2d_157">+ <path d="M 561.384 580.292471+L 943.92 580.292471+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_158">+ <g>+ <use xlink:href="#m48df5c1fc4" x="561.384" y="580.292471" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_59">+ <!-- 200 -->+ <g style="fill: #444444" transform="translate(527.57025 585.230947) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-15"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="ytick_16">+ <g id="line2d_159">+ <path d="M 561.384 517.980706+L 943.92 517.980706+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #e4e6e8; stroke-width: 0.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_160">+ <g>+ <use xlink:href="#m48df5c1fc4" x="561.384" y="517.980706" style="fill: #444444; stroke: #444444; stroke-width: 0.8"/>+ </g>+ </g>+ <g id="text_60">+ <!-- 300 -->+ <g style="fill: #444444" transform="translate(527.57025 522.919182) scale(0.13 -0.13)">+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(127.25 0)"/>+ </g>+ </g>+ </g>+ <g id="text_61">+ <!-- Per-request throughput (tokens/s) -->+ <g style="fill: #333333" transform="translate(513.206969 717.094594) rotate(-90) scale(0.14 -0.14)">+ <use xlink:href="#DejaVuSans-33"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(56.734375 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(118.265625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(152.984375 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(189.0625 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(227.96875 0)"/>+ <use xlink:href="#DejaVuSans-54" transform="translate(289.5 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(352.984375 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(416.359375 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(477.890625 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(529.984375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(569.1875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(600.96875 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(640.171875 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(703.546875 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(742.453125 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(803.640625 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(867.015625 0)"/>+ <use xlink:href="#DejaVuSans-4b" transform="translate(930.5 0)"/>+ <use xlink:href="#DejaVuSans-53" transform="translate(993.875 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(1057.359375 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(1120.734375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(1159.9375 0)"/>+ <use xlink:href="#DejaVuSans-b" transform="translate(1191.71875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(1230.734375 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(1269.9375 0)"/>+ <use xlink:href="#DejaVuSans-4e" transform="translate(1331.125 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(1385.46875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(1447 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1510.375 0)"/>+ <use xlink:href="#DejaVuSans-12" transform="translate(1562.46875 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(1596.15625 0)"/>+ <use xlink:href="#DejaVuSans-c" transform="translate(1648.25 0)"/>+ </g>+ </g>+ </g>+ <g id="patch_12">+ <path d="M 561.384 704.916+L 561.384 493.056+" style="fill: none; stroke: #888888; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>+ </g>+ <g id="patch_13">+ <path d="M 561.384 704.916+L 943.92 704.916+" style="fill: none; stroke: #888888; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>+ </g>+ <g id="line2d_161">+ <path d="M 597.259728 564.663793+L 673.013691 650.365503+L 695.539657 672.136827+L 712.218406 686.488797+L 699.867982 696.456815+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke-dasharray: 6.66,2.88; stroke-dashoffset: 0; stroke: #7c8594; stroke-width: 1.8"/>+ </g>+ <g id="line2d_162">+ <path d="M 601.075357 549.747013+L 717.774699 628.491974+L 769.996917 653.944174+L 814.213119 674.028263+L 850.20981 687.273302+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #4c78b8; stroke-width: 1.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_163">+ <path d="M 563.747404 695.676534+L 571.153366 700.141968+L 571.981861 702.326552+L 572.599651 703.545802+L 572.919817 704.211344+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #d9a441; stroke-width: 1.8; stroke-linecap: square"/>+ </g>+ <g id="line2d_164">+ <path d="M 600.414499 552.330563+L 738.645224 618.293097+L 797.838297 647.141498+L 858.191365 668.655509+L 911.576615 683.524759+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #bf525b; stroke-width: 1.8; stroke-linecap: square"/>+ </g>+ <g id="text_62">+ <!-- 1 -->+ <g style="fill: #5e8c56" transform="translate(609.038429 532.16321) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-14"/>+ </g>+ </g>+ <g id="text_63">+ <!-- 8 -->+ <g style="fill: #5e8c56" transform="translate(746.368968 610.962074) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-1b"/>+ </g>+ </g>+ <g id="text_64">+ <!-- 16 -->+ <g style="fill: #5e8c56" transform="translate(810.318168 639.31389) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-14"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(63.625 0)"/>+ </g>+ </g>+ <g id="text_65">+ <!-- 32 -->+ <g style="fill: #5e8c56" transform="translate(874.020103 661.332579) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-16"/>+ <use xlink:href="#DejaVuSans-15" transform="translate(63.625 0)"/>+ </g>+ </g>+ <g id="text_66">+ <!-- 64 -->+ <g style="fill: #5e8c56" transform="translate(928.671376 676.78596) scale(0.115 -0.115)">+ <use xlink:href="#DejaVuSans-19"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(63.625 0)"/>+ </g>+ </g>+ <g id="text_67">+ <!-- Qwen3-30B-A3B -->+ <g style="fill: #222222" transform="translate(687.41325 479.056) scale(0.16 -0.16)">+ <use xlink:href="#DejaVuSans-34"/>+ <use xlink:href="#DejaVuSans-5a" transform="translate(78.71875 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(160.5 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(222.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(285.40625 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(349.03125 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(385.109375 0)"/>+ <use xlink:href="#DejaVuSans-13" transform="translate(448.734375 0)"/>+ <use xlink:href="#DejaVuSans-25" transform="translate(512.359375 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(580.96875 0)"/>+ <use xlink:href="#DejaVuSans-24" transform="translate(614.84375 0)"/>+ <use xlink:href="#DejaVuSans-16" transform="translate(683.25 0)"/>+ <use xlink:href="#DejaVuSans-25" transform="translate(746.875 0)"/>+ </g>+ </g>+ <g id="line2d_165">+ <path d="M 597.259728 564.663793+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#m2c8cd094e3" x="597.259728" y="564.663793" style="fill: #7c8594; stroke: #ffffff; stroke-width: 0.6"/>+ </g>+ </g>+ <g id="line2d_166">+ <path d="M 673.013691 650.365503+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#m2c8cd094e3" x="673.013691" y="650.365503" style="fill: #7c8594; stroke: #ffffff; stroke-width: 0.6"/>+ </g>+ </g>+ <g id="line2d_167">+ <path d="M 695.539657 672.136827+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#m2c8cd094e3" x="695.539657" y="672.136827" style="fill: #7c8594; stroke: #ffffff; stroke-width: 0.6"/>+ </g>+ </g>+ <g id="line2d_168">+ <path d="M 712.218406 686.488797+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#m2c8cd094e3" x="712.218406" y="686.488797" style="fill: #7c8594; stroke: #ffffff; stroke-width: 0.6"/>+ </g>+ </g>+ <g id="line2d_169">+ <path d="M 699.867982 696.456815+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #7c8594; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#m3bfe823a7e" x="699.867982" y="696.456815" style="fill: #ffffff; stroke: #7c8594; stroke-width: 1.2"/>+ </g>+ </g>+ <g id="line2d_170">+ <path d="M 601.075357 549.747013+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#ma7f247753d" x="601.075357" y="549.747013" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_171">+ <path d="M 717.774699 628.491974+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#ma7f247753d" x="717.774699" y="628.491974" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_172">+ <path d="M 769.996917 653.944174+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#ma7f247753d" x="769.996917" y="653.944174" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_173">+ <path d="M 814.213119 674.028263+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#ma7f247753d" x="814.213119" y="674.028263" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_174">+ <path d="M 850.20981 687.273302+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #4c78b8; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#ma7f247753d" x="850.20981" y="687.273302" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_175">+ <path d="M 563.747404 695.676534+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#m5d68a1abc3" x="563.747404" y="695.676534" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_176">+ <path d="M 571.153366 700.141968+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#m5d68a1abc3" x="571.153366" y="700.141968" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_177">+ <path d="M 571.981861 702.326552+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#m5d68a1abc3" x="571.981861" y="702.326552" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_178">+ <path d="M 572.599651 703.545802+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#m5d68a1abc3" x="572.599651" y="703.545802" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_179">+ <path d="M 572.919817 704.211344+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #d9a441; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#m5d68a1abc3" x="572.919817" y="704.211344" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_180">+ <path d="M 600.414499 552.330563+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#mef82d167b8" x="600.414499" y="552.330563" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_181">+ <path d="M 738.645224 618.293097+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#mef82d167b8" x="738.645224" y="618.293097" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_182">+ <path d="M 797.838297 647.141498+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#mef82d167b8" x="797.838297" y="647.141498" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_183">+ <path d="M 858.191365 668.655509+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#mef82d167b8" x="858.191365" y="668.655509" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_184">+ <path d="M 911.576615 683.524759+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #bf525b; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#mef82d167b8" x="911.576615" y="683.524759" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_185">+ <path d="M 604.038429 538.16321+L 741.368968 616.962074+L 805.318168 645.31389+L 869.020103 667.332579+L 923.671376 682.78596+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #5e8c56; stroke-width: 2.3; stroke-linecap: square"/>+ </g>+ <g id="line2d_186">+ <path d="M 604.038429 538.16321+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#ma29a59a539" x="604.038429" y="538.16321" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_187">+ <path d="M 741.368968 616.962074+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#ma29a59a539" x="741.368968" y="616.962074" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_188">+ <path d="M 805.318168 645.31389+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#ma29a59a539" x="805.318168" y="645.31389" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_189">+ <path d="M 869.020103 667.332579+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#ma29a59a539" x="869.020103" y="667.332579" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="line2d_190">+ <path d="M 923.671376 682.78596+" clip-path="url(#p6ca8bc8a6c)" style="fill: none; stroke: #5e8c56; stroke-width: 1.5; stroke-linecap: square"/>+ <g clip-path="url(#p6ca8bc8a6c)">+ <use xlink:href="#ma29a59a539" x="923.671376" y="682.78596" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ </g>+ <g id="text_68">+ <!-- 64K input / 2K output · TP=1 -->+ <g style="fill: #222222" transform="translate(384.257422 91.47188) scale(0.17 -0.17)">+ <defs>+ <path id="DejaVuSans-Bold-19" d="M 2316 2303+Q 2000 2303 1842 2098+Q 1684 1894 1684 1484+Q 1684 1075 1842 870+Q 2000 666 2316 666+Q 2634 666 2792 870+Q 2950 1075 2950 1484+Q 2950 1894 2792 2098+Q 2634 2303 2316 2303+z+M 3803 4544+L 3803 3681+Q 3506 3822 3243 3889+Q 2981 3956 2731 3956+Q 2194 3956 1894 3657+Q 1594 3359 1544 2772+Q 1750 2925 1990 3001+Q 2231 3078 2516 3078+Q 3231 3078 3670 2659+Q 4109 2241 4109 1563+Q 4109 813 3618 361+Q 3128 -91 2303 -91+Q 1394 -91 895 523+Q 397 1138 397 2266+Q 397 3422 980 4083+Q 1563 4744 2578 4744+Q 2900 4744 3203 4694+Q 3506 4644 3803 4544+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-17" d="M 2356 3675+L 1038 1722+L 2356 1722+L 2356 3675+z+M 2156 4666+L 3494 4666+L 3494 1722+L 4159 1722+L 4159 850+L 3494 850+L 3494 0+L 2356 0+L 2356 850+L 288 850+L 288 1881+L 2156 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-2e" d="M 588 4666+L 1791 4666+L 1791 2963+L 3525 4666+L 4922 4666+L 2675 2456+L 5153 0+L 3647 0+L 1791 1838+L 1791 0+L 588 0+L 588 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-3" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-4c" d="M 538 3500+L 1656 3500+L 1656 0+L 538 0+L 538 3500+z+M 538 4863+L 1656 4863+L 1656 3950+L 538 3950+L 538 4863+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-51" d="M 4056 2131+L 4056 0+L 2931 0+L 2931 347+L 2931 1631+Q 2931 2084 2911 2256+Q 2891 2428 2841 2509+Q 2775 2619 2662 2680+Q 2550 2741 2406 2741+Q 2056 2741 1856 2470+Q 1656 2200 1656 1722+L 1656 0+L 538 0+L 538 3500+L 1656 3500+L 1656 2988+Q 1909 3294 2193 3439+Q 2478 3584 2822 3584+Q 3428 3584 3742 3212+Q 4056 2841 4056 2131+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-53" d="M 1656 506+L 1656 -1331+L 538 -1331+L 538 3500+L 1656 3500+L 1656 2988+Q 1888 3294 2169 3439+Q 2450 3584 2816 3584+Q 3463 3584 3878 3070+Q 4294 2556 4294 1747+Q 4294 938 3878 423+Q 3463 -91 2816 -91+Q 2450 -91 2169 54+Q 1888 200 1656 506+z+M 2400 2772+Q 2041 2772 1848 2508+Q 1656 2244 1656 1747+Q 1656 1250 1848 986+Q 2041 722 2400 722+Q 2759 722 2948 984+Q 3138 1247 3138 1747+Q 3138 2247 2948 2509+Q 2759 2772 2400 2772+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-58" d="M 500 1363+L 500 3500+L 1625 3500+L 1625 3150+Q 1625 2866 1622 2436+Q 1619 2006 1619 1863+Q 1619 1441 1641 1255+Q 1663 1069 1716 984+Q 1784 875 1895 815+Q 2006 756 2150 756+Q 2500 756 2700 1025+Q 2900 1294 2900 1772+L 2900 3500+L 4019 3500+L 4019 0+L 2900 0+L 2900 506+Q 2647 200 2364 54+Q 2081 -91 1741 -91+Q 1134 -91 817 281+Q 500 653 500 1363+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-57" d="M 1759 4494+L 1759 3500+L 2913 3500+L 2913 2700+L 1759 2700+L 1759 1216+Q 1759 972 1856 886+Q 1953 800 2241 800+L 2816 800+L 2816 0+L 1856 0+Q 1194 0 917 276+Q 641 553 641 1216+L 641 2700+L 84 2700+L 84 3500+L 641 3500+L 641 4494+L 1759 4494+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-12" d="M 1644 4666+L 2338 4666+L 691 -594+L 0 -594+L 1644 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-15" d="M 1844 884+L 3897 884+L 3897 0+L 506 0+L 506 884+L 2209 2388+Q 2438 2594 2547 2791+Q 2656 2988 2656 3200+Q 2656 3528 2436 3728+Q 2216 3928 1850 3928+Q 1569 3928 1234 3808+Q 900 3688 519 3450+L 519 4475+Q 925 4609 1322 4679+Q 1719 4750 2100 4750+Q 2938 4750 3402 4381+Q 3866 4013 3866 3353+Q 3866 2972 3669 2642+Q 3472 2313 2841 1759+L 1844 884+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-52" d="M 2203 2784+Q 1831 2784 1636 2517+Q 1441 2250 1441 1747+Q 1441 1244 1636 976+Q 1831 709 2203 709+Q 2569 709 2762 976+Q 2956 1244 2956 1747+Q 2956 2250 2762 2517+Q 2569 2784 2203 2784+z+M 2203 3584+Q 3106 3584 3614 3096+Q 4122 2609 4122 1747+Q 4122 884 3614 396+Q 3106 -91 2203 -91+Q 1297 -91 786 396+Q 275 884 275 1747+Q 275 2609 786 3096+Q 1297 3584 2203 3584+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-79" d="M 653 2828+L 1778 2828+L 1778 1619+L 653 1619+L 653 2828+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-37" d="M 31 4666+L 4331 4666+L 4331 3756+L 2784 3756+L 2784 0+L 1581 0+L 1581 3756+L 31 3756+L 31 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-33" d="M 588 4666+L 2584 4666+Q 3475 4666 3951 4270+Q 4428 3875 4428 3144+Q 4428 2409 3951 2014+Q 3475 1619 2584 1619+L 1791 1619+L 1791 0+L 588 0+L 588 4666+z+M 1791 3794+L 1791 2491+L 2456 2491+Q 2806 2491 2997 2661+Q 3188 2831 3188 3144+Q 3188 3456 2997 3625+Q 2806 3794 2456 3794+L 1791 3794+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-20" d="M 678 3084+L 4684 3084+L 4684 2350+L 678 2350+L 678 3084+z+M 678 1663+L 4684 1663+L 4684 922+L 678 922+L 678 1663+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-Bold-14" d="M 750 831+L 1813 831+L 1813 3847+L 722 3622+L 722 4441+L 1806 4666+L 2950 4666+L 2950 831+L 4013 831+L 4013 0+L 750 0+L 750 831+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-Bold-19"/>+ <use xlink:href="#DejaVuSans-Bold-17" transform="translate(69.578125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-2e" transform="translate(139.15625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-3" transform="translate(216.640625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-4c" transform="translate(251.453125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-51" transform="translate(285.734375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-53" transform="translate(356.921875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-58" transform="translate(428.5 0)"/>+ <use xlink:href="#DejaVuSans-Bold-57" transform="translate(499.6875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-3" transform="translate(547.484375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-12" transform="translate(582.296875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-3" transform="translate(618.828125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-15" transform="translate(653.640625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-2e" transform="translate(723.21875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-3" transform="translate(800.703125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-52" transform="translate(835.515625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-58" transform="translate(904.21875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-57" transform="translate(975.40625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-53" transform="translate(1023.203125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-58" transform="translate(1094.78125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-57" transform="translate(1165.96875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-3" transform="translate(1213.765625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-79" transform="translate(1248.578125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-3" transform="translate(1286.5625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-37" transform="translate(1321.375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-33" transform="translate(1389.59375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-20" transform="translate(1462.890625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-14" transform="translate(1546.6875 0)"/>+ </g>+ </g>+ <g id="text_69">+ <!-- 128K input / 2K output · TP=2 -->+ <g style="fill: #222222" transform="translate(378.343281 445.85588) scale(0.17 -0.17)">+ <defs>+ <path id="DejaVuSans-Bold-1b" d="M 2228 2088+Q 1891 2088 1709 1903+Q 1528 1719 1528 1375+Q 1528 1031 1709 848+Q 1891 666 2228 666+Q 2563 666 2741 848+Q 2919 1031 2919 1375+Q 2919 1722 2741 1905+Q 2563 2088 2228 2088+z+M 1350 2484+Q 925 2613 709 2878+Q 494 3144 494 3541+Q 494 4131 934 4440+Q 1375 4750 2228 4750+Q 3075 4750 3515 4442+Q 3956 4134 3956 3541+Q 3956 3144 3739 2878+Q 3522 2613 3097 2484+Q 3572 2353 3814 2058+Q 4056 1763 4056 1313+Q 4056 619 3595 264+Q 3134 -91 2228 -91+Q 1319 -91 855 264+Q 391 619 391 1313+Q 391 1763 633 2058+Q 875 2353 1350 2484+z+M 1631 3419+Q 1631 3141 1786 2991+Q 1941 2841 2228 2841+Q 2509 2841 2662 2991+Q 2816 3141 2816 3419+Q 2816 3697 2662 3845+Q 2509 3994 2228 3994+Q 1941 3994 1786 3844+Q 1631 3694 1631 3419+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-Bold-14"/>+ <use xlink:href="#DejaVuSans-Bold-15" transform="translate(69.578125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-1b" transform="translate(139.15625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-2e" transform="translate(208.734375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-3" transform="translate(286.21875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-4c" transform="translate(321.03125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-51" transform="translate(355.3125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-53" transform="translate(426.5 0)"/>+ <use xlink:href="#DejaVuSans-Bold-58" transform="translate(498.078125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-57" transform="translate(569.265625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-3" transform="translate(617.0625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-12" transform="translate(651.875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-3" transform="translate(688.40625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-15" transform="translate(723.21875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-2e" transform="translate(792.796875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-3" transform="translate(870.28125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-52" transform="translate(905.09375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-58" transform="translate(973.796875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-57" transform="translate(1044.984375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-53" transform="translate(1092.78125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-58" transform="translate(1164.359375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-57" transform="translate(1235.546875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-3" transform="translate(1283.34375 0)"/>+ <use xlink:href="#DejaVuSans-Bold-79" transform="translate(1318.15625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-3" transform="translate(1356.140625 0)"/>+ <use xlink:href="#DejaVuSans-Bold-37" transform="translate(1390.953125 0)"/>+ <use xlink:href="#DejaVuSans-Bold-33" transform="translate(1459.171875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-20" transform="translate(1532.46875 0)"/>+ <use xlink:href="#DejaVuSans-Bold-15" transform="translate(1616.265625 0)"/>+ </g>+ </g>+ <g id="legend_1">+ <g id="line2d_191">+ <path d="M 257.828594 20.649013+L 272.528594 20.649013+L 287.228594 20.649013+" style="fill: none; stroke-dasharray: 6.66,2.88; stroke-dashoffset: 0; stroke: #7c8594; stroke-width: 1.8"/>+ <defs>+ <path id="m3e809ed516" d="M 0 3.5+C 0.928211 3.5 1.81853 3.131218 2.474874 2.474874+C 3.131218 1.81853 3.5 0.928211 3.5 0+C 3.5 -0.928211 3.131218 -1.81853 2.474874 -2.474874+C 1.81853 -3.131218 0.928211 -3.5 0 -3.5+C -0.928211 -3.5 -1.81853 -3.131218 -2.474874 -2.474874+C -3.131218 -1.81853 -3.5 -0.928211 -3.5 0+C -3.5 0.928211 -3.131218 1.81853 -2.474874 2.474874+C -1.81853 3.131218 -0.928211 3.5 0 3.5+z+" style="stroke: #ffffff; stroke-width: 0.6"/>+ </defs>+ <g>+ <use xlink:href="#m3e809ed516" x="272.528594" y="20.649013" style="fill: #7c8594; stroke: #ffffff; stroke-width: 0.6"/>+ </g>+ </g>+ <g id="text_70">+ <!-- BF16 KV -->+ <g style="fill: #222222" transform="translate(298.428594 25.549013) scale(0.14 -0.14)">+ <defs>+ <path id="DejaVuSans-29" d="M 628 4666+L 3309 4666+L 3309 4134+L 1259 4134+L 1259 2759+L 3109 2759+L 3109 2228+L 1259 2228+L 1259 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-2e" d="M 628 4666+L 1259 4666+L 1259 2694+L 3353 4666+L 4166 4666+L 1850 2491+L 4331 0+L 3500 0+L 1259 2247+L 1259 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-39" d="M 1831 0+L 50 4666+L 709 4666+L 2188 738+L 3669 4666+L 4325 4666+L 2547 0+L 1831 0+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-25"/>+ <use xlink:href="#DejaVuSans-29" transform="translate(68.609375 0)"/>+ <use xlink:href="#DejaVuSans-14" transform="translate(126.125 0)"/>+ <use xlink:href="#DejaVuSans-19" transform="translate(189.75 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(253.375 0)"/>+ <use xlink:href="#DejaVuSans-2e" transform="translate(285.15625 0)"/>+ <use xlink:href="#DejaVuSans-39" transform="translate(350.734375 0)"/>+ </g>+ </g>+ <g id="line2d_192">+ <path d="M 257.828594 44.450106+L 272.528594 44.450106+L 287.228594 44.450106+" style="fill: none; stroke: #4c78b8; stroke-width: 1.8; stroke-linecap: square"/>+ <defs>+ <path id="md7e55d46fc" d="M 0 -3.5+L -3.5 3.5+L 3.5 3.5+z+" style="stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </defs>+ <g>+ <use xlink:href="#md7e55d46fc" x="272.528594" y="44.450106" style="fill: #4c78b8; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="text_71">+ <!-- FP8 KV -->+ <g style="fill: #222222" transform="translate(298.428594 49.350106) scale(0.14 -0.14)">+ <use xlink:href="#DejaVuSans-29"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(57.515625 0)"/>+ <use xlink:href="#DejaVuSans-1b" transform="translate(117.8125 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(181.4375 0)"/>+ <use xlink:href="#DejaVuSans-2e" transform="translate(213.21875 0)"/>+ <use xlink:href="#DejaVuSans-39" transform="translate(278.796875 0)"/>+ </g>+ </g>+ <g id="line2d_193">+ <path d="M 390.708281 20.649013+L 405.408281 20.649013+L 420.108281 20.649013+" style="fill: none; stroke: #d9a441; stroke-width: 1.8; stroke-linecap: square"/>+ <defs>+ <path id="mac4e95937e" d="M -0 3.5+L 3.5 -3.5+L -3.5 -3.5+z+" style="stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </defs>+ <g>+ <use xlink:href="#mac4e95937e" x="405.408281" y="20.649013" style="fill: #d9a441; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="text_72">+ <!-- TurboQuant-4bit-nc -->+ <g style="fill: #222222" transform="translate(431.308281 25.549013) scale(0.14 -0.14)">+ <defs>+ <path id="DejaVuSans-37" d="M -19 4666+L 3928 4666+L 3928 4134+L 2272 4134+L 2272 0+L 1638 0+L 1638 4134+L -19 4134+L -19 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-45" d="M 3116 1747+Q 3116 2381 2855 2742+Q 2594 3103 2138 3103+Q 1681 3103 1420 2742+Q 1159 2381 1159 1747+Q 1159 1113 1420 752+Q 1681 391 2138 391+Q 2594 391 2855 752+Q 3116 1113 3116 1747+z+M 1159 2969+Q 1341 3281 1617 3432+Q 1894 3584 2278 3584+Q 2916 3584 3314 3078+Q 3713 2572 3713 1747+Q 3713 922 3314 415+Q 2916 -91 2278 -91+Q 1894 -91 1617 61+Q 1341 213 1159 525+L 1159 0+L 581 0+L 581 4863+L 1159 4863+L 1159 2969+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-44" d="M 2194 1759+Q 1497 1759 1228 1600+Q 959 1441 959 1056+Q 959 750 1161 570+Q 1363 391 1709 391+Q 2188 391 2477 730+Q 2766 1069 2766 1631+L 2766 1759+L 2194 1759+z+M 3341 1997+L 3341 0+L 2766 0+L 2766 531+Q 2569 213 2275 61+Q 1981 -91 1556 -91+Q 1019 -91 701 211+Q 384 513 384 1019+Q 384 1609 779 1909+Q 1175 2209 1959 2209+L 2766 2209+L 2766 2266+Q 2766 2663 2505 2880+Q 2244 3097 1772 3097+Q 1472 3097 1187 3025+Q 903 2953 641 2809+L 641 3341+Q 956 3463 1253 3523+Q 1550 3584 1831 3584+Q 2591 3584 2966 3190+Q 3341 2797 3341 1997+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-4c" d="M 603 3500+L 1178 3500+L 1178 0+L 603 0+L 603 3500+z+M 603 4863+L 1178 4863+L 1178 4134+L 603 4134+L 603 4863+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-46" d="M 3122 3366+L 3122 2828+Q 2878 2963 2633 3030+Q 2388 3097 2138 3097+Q 1578 3097 1268 2742+Q 959 2388 959 1747+Q 959 1106 1268 751+Q 1578 397 2138 397+Q 2388 397 2633 464+Q 2878 531 3122 666+L 3122 134+Q 2881 22 2623 -34+Q 2366 -91 2075 -91+Q 1284 -91 818 406+Q 353 903 353 1747+Q 353 2603 823 3093+Q 1294 3584 2113 3584+Q 2378 3584 2631 3529+Q 2884 3475 3122 3366+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-37"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(45.890625 0)"/>+ <use xlink:href="#DejaVuSans-55" transform="translate(109.265625 0)"/>+ <use xlink:href="#DejaVuSans-45" transform="translate(150.375 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(213.859375 0)"/>+ <use xlink:href="#DejaVuSans-34" transform="translate(275.046875 0)"/>+ <use xlink:href="#DejaVuSans-58" transform="translate(353.765625 0)"/>+ <use xlink:href="#DejaVuSans-44" transform="translate(417.140625 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(478.421875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(541.796875 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(581 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(617.078125 0)"/>+ <use xlink:href="#DejaVuSans-45" transform="translate(680.703125 0)"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(744.1875 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(771.96875 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(811.171875 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(847.25 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(910.625 0)"/>+ </g>+ </g>+ <g id="line2d_194">+ <path d="M 390.708281 44.450106+L 405.408281 44.450106+L 420.108281 44.450106+" style="fill: none; stroke: #bf525b; stroke-width: 1.8; stroke-linecap: square"/>+ <defs>+ <path id="m195780b777" d="M -0 4.949747+L 4.949747 0+L 0 -4.949747+L -4.949747 -0+z+" style="stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </defs>+ <g>+ <use xlink:href="#m195780b777" x="405.408281" y="44.450106" style="fill: #bf525b; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="text_73">+ <!-- TRT-LLM-NVFP4 -->+ <g style="fill: #222222" transform="translate(431.308281 49.350106) scale(0.14 -0.14)">+ <defs>+ <path id="DejaVuSans-35" d="M 2841 2188+Q 3044 2119 3236 1894+Q 3428 1669 3622 1275+L 4263 0+L 3584 0+L 2988 1197+Q 2756 1666 2539 1819+Q 2322 1972 1947 1972+L 1259 1972+L 1259 0+L 628 0+L 628 4666+L 2053 4666+Q 2853 4666 3247 4331+Q 3641 3997 3641 3322+Q 3641 2881 3436 2590+Q 3231 2300 2841 2188+z+M 1259 4147+L 1259 2491+L 2053 2491+Q 2509 2491 2742 2702+Q 2975 2913 2975 3322+Q 2975 3731 2742 3939+Q 2509 4147 2053 4147+L 1259 4147+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-2f" d="M 628 4666+L 1259 4666+L 1259 531+L 3531 531+L 3531 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-30" d="M 628 4666+L 1569 4666+L 2759 1491+L 3956 4666+L 4897 4666+L 4897 0+L 4281 0+L 4281 4097+L 3078 897+L 2444 897+L 1241 4097+L 1241 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-31" d="M 628 4666+L 1478 4666+L 3547 763+L 3547 4666+L 4159 4666+L 4159 0+L 3309 0+L 1241 3903+L 1241 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-37"/>+ <use xlink:href="#DejaVuSans-35" transform="translate(61.078125 0)"/>+ <use xlink:href="#DejaVuSans-37" transform="translate(123.28125 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(175.1875 0)"/>+ <use xlink:href="#DejaVuSans-2f" transform="translate(211.265625 0)"/>+ <use xlink:href="#DejaVuSans-2f" transform="translate(266.984375 0)"/>+ <use xlink:href="#DejaVuSans-30" transform="translate(322.703125 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(408.984375 0)"/>+ <use xlink:href="#DejaVuSans-31" transform="translate(445.0625 0)"/>+ <use xlink:href="#DejaVuSans-39" transform="translate(519.875 0)"/>+ <use xlink:href="#DejaVuSans-29" transform="translate(588.28125 0)"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(645.796875 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(706.09375 0)"/>+ </g>+ </g>+ <g id="line2d_195">+ <path d="M 600.093594 20.649013+L 614.793594 20.649013+L 629.493594 20.649013+" style="fill: none; stroke: #5e8c56; stroke-width: 2.3; stroke-linecap: square"/>+ <g>+ <use xlink:href="#ma29a59a539" x="614.793594" y="20.649013" style="fill: #5e8c56; stroke: #ffffff; stroke-width: 0.6; stroke-linejoin: miter"/>+ </g>+ </g>+ <g id="text_74">+ <!-- BitDecoding-NVFP4 -->+ <g style="fill: #222222" transform="translate(640.693594 25.549013) scale(0.14 -0.14)">+ <defs>+ <path id="DejaVuSans-27" d="M 1259 4147+L 1259 519+L 2022 519+Q 2988 519 3436 956+Q 3884 1394 3884 2338+Q 3884 3275 3436 3711+Q 2988 4147 2022 4147+L 1259 4147+z+M 628 4666+L 1925 4666+Q 3281 4666 3915 4102+Q 4550 3538 4550 2338+Q 4550 1131 3912 565+Q 3275 0 1925 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-47" d="M 2906 2969+L 2906 4863+L 3481 4863+L 3481 0+L 2906 0+L 2906 525+Q 2725 213 2448 61+Q 2172 -91 1784 -91+Q 1150 -91 751 415+Q 353 922 353 1747+Q 353 2572 751 3078+Q 1150 3584 1784 3584+Q 2172 3584 2448 3432+Q 2725 3281 2906 2969+z+M 947 1747+Q 947 1113 1208 752+Q 1469 391 1925 391+Q 2381 391 2643 752+Q 2906 1113 2906 1747+Q 2906 2381 2643 2742+Q 2381 3103 1925 3103+Q 1469 3103 1208 2742+Q 947 2381 947 1747+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-25"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(68.609375 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(96.390625 0)"/>+ <use xlink:href="#DejaVuSans-27" transform="translate(135.59375 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(212.59375 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(274.125 0)"/>+ <use xlink:href="#DejaVuSans-52" transform="translate(329.109375 0)"/>+ <use xlink:href="#DejaVuSans-47" transform="translate(390.296875 0)"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(453.78125 0)"/>+ <use xlink:href="#DejaVuSans-51" transform="translate(481.5625 0)"/>+ <use xlink:href="#DejaVuSans-4a" transform="translate(544.9375 0)"/>+ <use xlink:href="#DejaVuSans-10" transform="translate(608.421875 0)"/>+ <use xlink:href="#DejaVuSans-31" transform="translate(644.5 0)"/>+ <use xlink:href="#DejaVuSans-39" transform="translate(719.3125 0)"/>+ <use xlink:href="#DejaVuSans-29" transform="translate(787.71875 0)"/>+ <use xlink:href="#DejaVuSans-33" transform="translate(845.234375 0)"/>+ <use xlink:href="#DejaVuSans-17" transform="translate(905.53125 0)"/>+ </g>+ </g>+ <g id="line2d_196">+ <defs>+ <path id="me0673129be" d="M 0 3.5+C 0.928211 3.5 1.81853 3.131218 2.474874 2.474874+C 3.131218 1.81853 3.5 0.928211 3.5 0+C 3.5 -0.928211 3.131218 -1.81853 2.474874 -2.474874+C 1.81853 -3.131218 0.928211 -3.5 0 -3.5+C -0.928211 -3.5 -1.81853 -3.131218 -2.474874 -2.474874+C -3.131218 -1.81853 -3.5 -0.928211 -3.5 0+C -3.5 0.928211 -3.131218 1.81853 -2.474874 2.474874+C -1.81853 3.131218 -0.928211 3.5 0 3.5+z+" style="stroke: #555555; stroke-width: 1.2"/>+ </defs>+ <g>+ <use xlink:href="#me0673129be" x="614.793594" y="44.450106" style="fill: #ffffff; stroke: #555555; stroke-width: 1.2"/>+ </g>+ </g>+ <g id="text_75">+ <!-- Exceeds KV capacity -->+ <g style="fill: #222222" transform="translate(640.693594 49.350106) scale(0.14 -0.14)">+ <defs>+ <path id="DejaVuSans-28" d="M 628 4666+L 3578 4666+L 3578 4134+L 1259 4134+L 1259 2753+L 3481 2753+L 3481 2222+L 1259 2222+L 1259 531+L 3634 531+L 3634 0+L 628 0+L 628 4666+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-5b" d="M 3513 3500+L 2247 1797+L 3578 0+L 2900 0+L 1881 1375+L 863 0+L 184 0+L 1544 1831+L 300 3500+L 978 3500+L 1906 2253+L 2834 3500+L 3513 3500+z+" transform="scale(0.015625)"/>+ <path id="DejaVuSans-5c" d="M 2059 -325+Q 1816 -950 1584 -1140+Q 1353 -1331 966 -1331+L 506 -1331+L 506 -850+L 844 -850+Q 1081 -850 1212 -737+Q 1344 -625 1503 -206+L 1606 56+L 191 3500+L 800 3500+L 1894 763+L 2988 3500+L 3597 3500+L 2059 -325+z+" transform="scale(0.015625)"/>+ </defs>+ <use xlink:href="#DejaVuSans-28"/>+ <use xlink:href="#DejaVuSans-5b" transform="translate(63.1875 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(120.625 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(175.609375 0)"/>+ <use xlink:href="#DejaVuSans-48" transform="translate(237.140625 0)"/>+ <use xlink:href="#DejaVuSans-47" transform="translate(298.671875 0)"/>+ <use xlink:href="#DejaVuSans-56" transform="translate(362.15625 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(414.25 0)"/>+ <use xlink:href="#DejaVuSans-2e" transform="translate(446.03125 0)"/>+ <use xlink:href="#DejaVuSans-39" transform="translate(511.609375 0)"/>+ <use xlink:href="#DejaVuSans-3" transform="translate(580.015625 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(611.796875 0)"/>+ <use xlink:href="#DejaVuSans-44" transform="translate(666.78125 0)"/>+ <use xlink:href="#DejaVuSans-53" transform="translate(728.0625 0)"/>+ <use xlink:href="#DejaVuSans-44" transform="translate(791.546875 0)"/>+ <use xlink:href="#DejaVuSans-46" transform="translate(852.828125 0)"/>+ <use xlink:href="#DejaVuSans-4c" transform="translate(907.8125 0)"/>+ <use xlink:href="#DejaVuSans-57" transform="translate(935.59375 0)"/>+ <use xlink:href="#DejaVuSans-5c" transform="translate(974.796875 0)"/>+ </g>+ </g>+ </g>+ </g>+ <defs>+ <clipPath id="p4105837502">+ <rect x="84.456" y="138.672" width="382.536" height="211.86"/>+ </clipPath>+ <clipPath id="p78789ce146">+ <rect x="561.384" y="138.672" width="382.536" height="211.86"/>+ </clipPath>+ <clipPath id="p9992f1a396">+ <rect x="84.456" y="493.056" width="382.536" height="211.86"/>+ </clipPath>+ <clipPath id="p6ca8bc8a6c">+ <rect x="561.384" y="493.056" width="382.536" height="211.86"/>+ </clipPath>+ </defs>+</svg>diff --git a/imgs/icon.svg b/imgs/icon.svgnew file mode 100644index 0000000..75f5ac9--- /dev/null+++ b/imgs/icon.svg@@ -0,0 +1 @@+<svg xmlns="http://www.w3.org/2000/svg" viewBox="-80.25 -48 415.5 415.5" width="415.5" height="415.5" role="img" aria-label="BitDecoding-FP4"><title>BitDecoding-FP4</title><defs><clipPath id="clip-b"><path d="M0,0H81.5A36,36 0 0 1 117.5,36V78.75A20,20 0 0 1 97.5,98.75H0V0ZM0,98.75H97.5A20,20 0 0 1 117.5,118.75V165.5A32,32 0 0 1 85.5,197.5H0V98.75Z"/></clipPath><clipPath id="clip-d"><path d="M137.5,0H211A44,44 0 0 1 255,44V157.5A40,40 0 0 1 215,197.5H137.5V0Z"/></clipPath><clipPath id="clip-fp4"><path d="M-0.04 339.5V252.0H36.92V264.15H17.78V287.71H34.01V300.17H17.78V339.5ZM49.87 339.5V252.0H77.62Q85.51 252.0 90.505 254.95Q95.5 257.9 97.91499999999999 263.51Q100.33 269.12 100.33 277.13Q100.33 286.6 97.18 291.89Q94.03 297.18 88.50999999999999 299.38Q82.99 301.58 75.86 301.58H67.69V339.5ZM67.69 289.09H73.99Q77.92 289.09 80.00999999999999 287.815Q82.1 286.54 82.845 283.87Q83.59 281.2 83.59 276.9Q83.59 272.93 83.01 270.22Q82.43 267.51 80.36500000000001 266.04499999999996Q78.3 264.58 73.93 264.58H67.69ZM136.57 339.5V319.03H110.0V306.0L132.79 252.0H152.18V306.31H160.65V319.03H152.18V339.5ZM123.16 306.31H136.57V269.19Z"/></clipPath><linearGradient id="g-bit" gradientUnits="userSpaceOnUse" x1="0" y1="-20" x2="117.5" y2="219.5"><stop offset="0" stop-color="#4c78b8"/><stop offset="0.55" stop-color="#3d6a99"/><stop offset="1" stop-color="#6e4f96"/></linearGradient><linearGradient id="g-dec" gradientUnits="userSpaceOnUse" x1="0" y1="0" x2="0" y2="219.5"><stop offset="0" stop-color="#8a63b0"/><stop offset="0.6" stop-color="#bf525b"/><stop offset="1" stop-color="#d9a441"/></linearGradient><linearGradient id="g-fp4" gradientUnits="userSpaceOnUse" x1="0" y1="252" x2="162.58" y2="339.5"><stop offset="0" stop-color="#6c9a5b"/><stop offset="0.5" stop-color="#5e8c56"/><stop offset="1" stop-color="#3f6b39"/></linearGradient></defs><g fill="url(#g-bit)"><g clip-path="url(#clip-b)"><rect x="0" y="0" width="7.5" height="7.5" rx="1"/><rect x="0" y="10" width="7.5" height="7.5" rx="1"/><rect x="0" y="20" width="7.5" height="7.5" rx="1"/><rect x="0" y="30" width="7.5" height="7.5" rx="1"/><rect x="10" y="0" width="7.5" height="7.5" rx="1"/><rect x="10" y="10" width="7.5" height="7.5" rx="1"/><rect x="10" y="20" width="7.5" height="7.5" rx="1"/><rect x="10" y="30" width="7.5" height="7.5" rx="1"/><rect x="20" y="0" width="7.5" height="7.5" rx="1"/><rect x="20" y="10" width="7.5" height="7.5" rx="1"/><rect x="20" y="20" width="7.5" height="7.5" rx="1"/><rect x="20" y="30" width="7.5" height="7.5" rx="1"/><rect x="30" y="0" width="7.5" height="7.5" rx="1"/><rect x="30" y="10" width="7.5" height="7.5" rx="1"/><rect x="30" y="20" width="7.5" height="7.5" rx="1"/><rect x="30" y="30" width="7.5" height="7.5" rx="1"/><rect x="40" y="0" width="7.5" height="7.5" rx="1"/><rect x="40" y="10" width="7.5" height="7.5" rx="1"/><rect x="40" y="20" width="7.5" height="7.5" rx="1"/><rect x="40" y="30" width="7.5" height="7.5" rx="1"/><rect x="50" y="0" width="7.5" height="7.5" rx="1"/><rect x="50" y="10" width="7.5" height="7.5" rx="1"/><rect x="50" y="20" width="7.5" height="7.5" rx="1"/><rect x="50" y="30" width="7.5" height="7.5" rx="1"/><rect x="60" y="0" width="7.5" height="7.5" rx="1"/><rect x="60" y="10" width="7.5" height="7.5" rx="1"/><rect x="60" y="20" width="7.5" height="7.5" rx="1"/><rect x="60" y="30" width="7.5" height="7.5" rx="1"/><rect x="70" y="0" width="7.5" height="7.5" rx="1"/><rect x="70" y="10" width="7.5" height="7.5" rx="1"/><rect x="70" y="20" width="7.5" height="7.5" rx="1"/><rect x="70" y="30" width="7.5" height="7.5" rx="1"/><rect x="80" y="0" width="7.5" height="7.5" rx="1"/><rect x="80" y="10" width="7.5" height="7.5" rx="1"/><rect x="80" y="20" width="7.5" height="7.5" rx="1"/><rect x="80" y="30" width="7.5" height="7.5" rx="1"/><rect x="90" y="0" width="7.5" height="7.5" rx="1"/><rect x="90" y="10" width="7.5" height="7.5" rx="1"/><rect x="90" y="20" width="7.5" height="7.5" rx="1"/><rect x="90" y="30" width="7.5" height="7.5" rx="1"/><rect x="100" y="10" width="7.5" height="7.5" rx="1"/><rect x="100" y="20" width="7.5" height="7.5" rx="1"/><rect x="100" y="30" width="7.5" height="7.5" rx="1"/><rect x="110" y="20" width="7.5" height="7.5" rx="1"/><rect x="110" y="30" width="7.5" height="7.5" rx="1"/><rect x="0" y="40" width="37.5" height="37.5" rx="2"/><rect x="80" y="40" width="7.5" height="7.5" rx="1"/><rect x="80" y="50" width="7.5" height="7.5" rx="1"/><rect x="80" y="60" width="7.5" height="7.5" rx="1"/><rect x="80" y="70" width="7.5" height="7.5" rx="1"/><rect x="90" y="40" width="7.5" height="7.5" rx="1"/><rect x="90" y="50" width="7.5" height="7.5" rx="1"/><rect x="90" y="60" width="7.5" height="7.5" rx="1"/><rect x="90" y="70" width="7.5" height="7.5" rx="1"/><rect x="100" y="40" width="7.5" height="7.5" rx="1"/><rect x="100" y="50" width="7.5" height="7.5" rx="1"/><rect x="100" y="60" width="7.5" height="7.5" rx="1"/><rect x="100" y="70" width="7.5" height="7.5" rx="1"/><rect x="110" y="40" width="7.5" height="7.5" rx="1"/><rect x="110" y="50" width="7.5" height="7.5" rx="1"/><rect x="110" y="60" width="7.5" height="7.5" rx="1"/><rect x="110" y="70" width="7.5" height="7.5" rx="1"/><rect x="0" y="80" width="37.5" height="37.5" rx="2"/><rect x="40" y="80" width="37.5" height="37.5" rx="2"/><rect x="80" y="80" width="37.5" height="37.5" rx="2"/><rect x="0" y="120" width="37.5" height="37.5" rx="2"/><rect x="80" y="120" width="37.5" height="37.5" rx="2"/><rect x="0" y="160" width="37.5" height="37.5" rx="2"/><rect x="40" y="160" width="37.5" height="37.5" rx="2"/><rect x="80" y="160" width="37.5" height="37.5" rx="2"/></g><rect x="10" y="200" width="17.5" height="22" rx="1.5"/><rect x="50" y="200" width="17.5" height="22" rx="1.5"/><rect x="10" y="-10" width="7.5" height="7.5" rx="1"/><rect x="10" y="-20" width="7.5" height="7.5" rx="1"/><rect x="20" y="-10" width="7.5" height="7.5" rx="1"/><rect x="20" y="-20" width="7.5" height="7.5" rx="1"/><rect x="50" y="-10" width="7.5" height="7.5" rx="1"/><rect x="50" y="-20" width="7.5" height="7.5" rx="1"/><rect x="60" y="-10" width="7.5" height="7.5" rx="1"/><rect x="60" y="-20" width="7.5" height="7.5" rx="1"/></g><g fill="url(#g-dec)"><g clip-path="url(#clip-d)"><rect x="137.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="137.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="137.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="137.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="147.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="147.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="147.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="147.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="157.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="157.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="157.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="157.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="167.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="167.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="167.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="167.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="177.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="177.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="177.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="177.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="187.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="187.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="187.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="187.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="197.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="197.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="197.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="197.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="207.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="207.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="207.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="207.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="217.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="217.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="217.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="217.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="227.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="227.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="227.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="237.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="237.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="237.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="247.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="137.5" y="40" width="37.5" height="37.5" rx="2"/><rect x="217.5" y="40" width="7.5" height="7.5" rx="1"/><rect x="217.5" y="50" width="7.5" height="7.5" rx="1"/><rect x="217.5" y="60" width="7.5" height="7.5" rx="1"/><rect x="217.5" y="70" width="7.5" height="7.5" rx="1"/><rect x="227.5" y="40" width="7.5" height="7.5" rx="1"/><rect x="227.5" y="50" width="7.5" height="7.5" rx="1"/><rect x="227.5" y="60" width="7.5" height="7.5" rx="1"/><rect x="227.5" y="70" width="7.5" height="7.5" rx="1"/><rect x="237.5" y="40" width="7.5" height="7.5" rx="1"/><rect x="237.5" y="50" width="7.5" height="7.5" rx="1"/><rect x="237.5" y="60" width="7.5" height="7.5" rx="1"/><rect x="237.5" y="70" width="7.5" height="7.5" rx="1"/><rect x="247.5" y="40" width="7.5" height="7.5" rx="1"/><rect x="247.5" y="50" width="7.5" height="7.5" rx="1"/><rect x="247.5" y="60" width="7.5" height="7.5" rx="1"/><rect x="247.5" y="70" width="7.5" height="7.5" rx="1"/><rect x="137.5" y="80" width="37.5" height="37.5" rx="2"/><rect x="217.5" y="80" width="37.5" height="37.5" rx="2"/><rect x="137.5" y="120" width="37.5" height="37.5" rx="2"/><rect x="217.5" y="120" width="37.5" height="37.5" rx="2"/><rect x="137.5" y="160" width="37.5" height="37.5" rx="2"/><rect x="177.5" y="160" width="37.5" height="37.5" rx="2"/><rect x="217.5" y="160" width="37.5" height="37.5" rx="2"/></g><rect x="147.5" y="200" width="17.5" height="22" rx="1.5"/><rect x="187.5" y="200" width="17.5" height="22" rx="1.5"/></g><g fill="url(#g-fp4)" transform="translate(46.21,0)"><rect x="0" y="252" width="7.5" height="7.5" rx="1"/><rect x="0" y="262" width="7.5" height="7.5" rx="1"/><rect x="10" y="252" width="7.5" height="7.5" rx="1"/><rect x="10" y="262" width="7.5" height="7.5" rx="1"/><rect x="20" y="252" width="7.5" height="7.5" rx="1"/><rect x="30" y="252" width="7.5" height="7.5" rx="1"/><rect x="50" y="252" width="7.5" height="7.5" rx="1"/><rect x="50" y="262" width="7.5" height="7.5" rx="1"/><rect x="60" y="252" width="7.5" height="7.5" rx="1"/><rect x="60" y="262" width="7.5" height="7.5" rx="1"/><rect x="70" y="252" width="7.5" height="7.5" rx="1"/><rect x="80" y="252" width="7.5" height="7.5" rx="1"/><rect x="80" y="262" width="7.5" height="7.5" rx="1"/><rect x="90" y="262" width="7.5" height="7.5" rx="1"/><rect x="130" y="252" width="7.5" height="7.5" rx="1"/><rect x="130" y="262" width="7.5" height="7.5" rx="1"/><rect x="140" y="252" width="7.5" height="7.5" rx="1"/><rect x="140" y="262" width="7.5" height="7.5" rx="1"/><g clip-path="url(#clip-fp4)"><rect x="-4" y="272" width="170.58" height="68.5"/></g></g></svg>diff --git a/imgs/title.svg b/imgs/title.svgnew file mode 100644index 0000000..7a16bbb--- /dev/null+++ b/imgs/title.svg@@ -0,0 +1 @@+<svg xmlns="http://www.w3.org/2000/svg" viewBox="-12 -32 1313.67 278.64" width="1313.67" height="278.64" role="img" aria-label="BitDecoding-FP4"><title>BitDecoding-FP4</title><defs><clipPath id="clip-b"><path d="M0,0H81.5A36,36 0 0 1 117.5,36V78.75A20,20 0 0 1 97.5,98.75H0V0ZM0,98.75H97.5A20,20 0 0 1 117.5,118.75V165.5A32,32 0 0 1 85.5,197.5H0V98.75Z"/></clipPath><clipPath id="clip-d"><path d="M227.5,0H301A44,44 0 0 1 345,44V157.5A40,40 0 0 1 305,197.5H227.5V0Z"/></clipPath><clipPath id="clip-fp4"><path d="M1003.35 197.5V40.0H1069.87V61.87H1035.42V104.27H1064.64V126.71H1035.42V197.5ZM1089.99 197.5V40.0H1139.94Q1154.13 40.0 1163.125 45.305Q1172.12 50.61 1176.465 60.714999999999996Q1180.81 70.82 1180.81 85.24Q1180.81 102.28 1175.135 111.8Q1169.46 121.32 1159.53 125.285Q1149.6 129.25 1136.76 129.25H1122.06V197.5ZM1122.06 106.76H1133.39Q1140.47 106.76 1144.2350000000001 104.47Q1148.0 102.18 1149.335 97.36500000000001Q1150.67 92.55 1150.67 84.82Q1150.67 77.68 1149.63 72.80000000000001Q1148.59 67.92 1144.87 65.28Q1141.15 62.64 1133.28 62.64H1122.06ZM1242.85 197.5V160.66H1195.01V137.2L1236.04 40.0H1270.94V137.75H1286.19V160.66H1270.94V197.5ZM1218.71 137.75H1242.85V70.94Z"/></clipPath><linearGradient id="g-bit" gradientUnits="userSpaceOnUse" x1="0" y1="-20" x2="205.5" y2="219.5"><stop offset="0" stop-color="#4c78b8"/><stop offset="0.55" stop-color="#3d6a99"/><stop offset="1" stop-color="#6e4f96"/></linearGradient><linearGradient id="g-dec" gradientUnits="userSpaceOnUse" x1="0" y1="0" x2="0" y2="219.5"><stop offset="0" stop-color="#8a63b0"/><stop offset="0.6" stop-color="#bf525b"/><stop offset="1" stop-color="#d9a441"/></linearGradient><linearGradient id="g-fp4" gradientUnits="userSpaceOnUse" x1="1003.42" y1="0" x2="1289.67" y2="219.5"><stop offset="0" stop-color="#6c9a5b"/><stop offset="0.5" stop-color="#5e8c56"/><stop offset="1" stop-color="#3f6b39"/></linearGradient></defs><g fill="url(#g-bit)"><g clip-path="url(#clip-b)"><rect x="0" y="0" width="7.5" height="7.5" rx="1"/><rect x="0" y="10" width="7.5" height="7.5" rx="1"/><rect x="0" y="20" width="7.5" height="7.5" rx="1"/><rect x="0" y="30" width="7.5" height="7.5" rx="1"/><rect x="10" y="0" width="7.5" height="7.5" rx="1"/><rect x="10" y="10" width="7.5" height="7.5" rx="1"/><rect x="10" y="20" width="7.5" height="7.5" rx="1"/><rect x="10" y="30" width="7.5" height="7.5" rx="1"/><rect x="20" y="0" width="7.5" height="7.5" rx="1"/><rect x="20" y="10" width="7.5" height="7.5" rx="1"/><rect x="20" y="20" width="7.5" height="7.5" rx="1"/><rect x="20" y="30" width="7.5" height="7.5" rx="1"/><rect x="30" y="0" width="7.5" height="7.5" rx="1"/><rect x="30" y="10" width="7.5" height="7.5" rx="1"/><rect x="30" y="20" width="7.5" height="7.5" rx="1"/><rect x="30" y="30" width="7.5" height="7.5" rx="1"/><rect x="40" y="0" width="7.5" height="7.5" rx="1"/><rect x="40" y="10" width="7.5" height="7.5" rx="1"/><rect x="40" y="20" width="7.5" height="7.5" rx="1"/><rect x="40" y="30" width="7.5" height="7.5" rx="1"/><rect x="50" y="0" width="7.5" height="7.5" rx="1"/><rect x="50" y="10" width="7.5" height="7.5" rx="1"/><rect x="50" y="20" width="7.5" height="7.5" rx="1"/><rect x="50" y="30" width="7.5" height="7.5" rx="1"/><rect x="60" y="0" width="7.5" height="7.5" rx="1"/><rect x="60" y="10" width="7.5" height="7.5" rx="1"/><rect x="60" y="20" width="7.5" height="7.5" rx="1"/><rect x="60" y="30" width="7.5" height="7.5" rx="1"/><rect x="70" y="0" width="7.5" height="7.5" rx="1"/><rect x="70" y="10" width="7.5" height="7.5" rx="1"/><rect x="70" y="20" width="7.5" height="7.5" rx="1"/><rect x="70" y="30" width="7.5" height="7.5" rx="1"/><rect x="80" y="0" width="7.5" height="7.5" rx="1"/><rect x="80" y="10" width="7.5" height="7.5" rx="1"/><rect x="80" y="20" width="7.5" height="7.5" rx="1"/><rect x="80" y="30" width="7.5" height="7.5" rx="1"/><rect x="90" y="0" width="7.5" height="7.5" rx="1"/><rect x="90" y="10" width="7.5" height="7.5" rx="1"/><rect x="90" y="20" width="7.5" height="7.5" rx="1"/><rect x="90" y="30" width="7.5" height="7.5" rx="1"/><rect x="100" y="10" width="7.5" height="7.5" rx="1"/><rect x="100" y="20" width="7.5" height="7.5" rx="1"/><rect x="100" y="30" width="7.5" height="7.5" rx="1"/><rect x="110" y="20" width="7.5" height="7.5" rx="1"/><rect x="110" y="30" width="7.5" height="7.5" rx="1"/><rect x="0" y="40" width="37.5" height="37.5" rx="2"/><rect x="80" y="40" width="7.5" height="7.5" rx="1"/><rect x="80" y="50" width="7.5" height="7.5" rx="1"/><rect x="80" y="60" width="7.5" height="7.5" rx="1"/><rect x="80" y="70" width="7.5" height="7.5" rx="1"/><rect x="90" y="40" width="7.5" height="7.5" rx="1"/><rect x="90" y="50" width="7.5" height="7.5" rx="1"/><rect x="90" y="60" width="7.5" height="7.5" rx="1"/><rect x="90" y="70" width="7.5" height="7.5" rx="1"/><rect x="100" y="40" width="7.5" height="7.5" rx="1"/><rect x="100" y="50" width="7.5" height="7.5" rx="1"/><rect x="100" y="60" width="7.5" height="7.5" rx="1"/><rect x="100" y="70" width="7.5" height="7.5" rx="1"/><rect x="110" y="40" width="7.5" height="7.5" rx="1"/><rect x="110" y="50" width="7.5" height="7.5" rx="1"/><rect x="110" y="60" width="7.5" height="7.5" rx="1"/><rect x="110" y="70" width="7.5" height="7.5" rx="1"/><rect x="0" y="80" width="37.5" height="37.5" rx="2"/><rect x="40" y="80" width="37.5" height="37.5" rx="2"/><rect x="80" y="80" width="37.5" height="37.5" rx="2"/><rect x="0" y="120" width="37.5" height="37.5" rx="2"/><rect x="80" y="120" width="37.5" height="37.5" rx="2"/><rect x="0" y="160" width="37.5" height="37.5" rx="2"/><rect x="40" y="160" width="37.5" height="37.5" rx="2"/><rect x="80" y="160" width="37.5" height="37.5" rx="2"/></g><rect x="10" y="200" width="17.5" height="22" rx="1.5"/><rect x="50" y="200" width="17.5" height="22" rx="1.5"/><rect x="10" y="-10" width="7.5" height="7.5" rx="1"/><rect x="10" y="-20" width="7.5" height="7.5" rx="1"/><rect x="20" y="-10" width="7.5" height="7.5" rx="1"/><rect x="20" y="-20" width="7.5" height="7.5" rx="1"/><rect x="50" y="-10" width="7.5" height="7.5" rx="1"/><rect x="50" y="-20" width="7.5" height="7.5" rx="1"/><rect x="60" y="-10" width="7.5" height="7.5" rx="1"/><rect x="60" y="-20" width="7.5" height="7.5" rx="1"/><rect x="139.5" y="85.11" width="7.5" height="7.5" rx="1"/><rect x="149.5" y="85.11" width="7.5" height="7.5" rx="1"/><rect x="139.5" y="55.11" width="7.5" height="7.5" rx="1"/><rect x="139.5" y="65.11" width="7.5" height="7.5" rx="1"/><rect x="149.5" y="55.11" width="7.5" height="7.5" rx="1"/><rect x="149.5" y="65.11" width="7.5" height="7.5" rx="1"/><rect x="139.5" y="95.11" width="17.5" height="102.39" rx="1.5"/><rect x="178" y="75.11" width="7.5" height="7.5" rx="1"/><rect x="178" y="65.11" width="7.5" height="7.5" rx="1"/><rect x="178" y="55.11" width="7.5" height="7.5" rx="1"/><rect x="188" y="75.11" width="7.5" height="7.5" rx="1"/><rect x="188" y="65.11" width="7.5" height="7.5" rx="1"/><rect x="188" y="55.11" width="7.5" height="7.5" rx="1"/><rect x="168" y="85.11" width="7.5" height="7.5" rx="1"/><rect x="178" y="85.11" width="7.5" height="7.5" rx="1"/><rect x="188" y="85.11" width="7.5" height="7.5" rx="1"/><rect x="198" y="85.11" width="7.5" height="7.5" rx="1"/><path d="M178,95.11H195.5V178.5A6,6 0 0 0 201.5,184.5H205.5V197.5H194A16,16 0 0 1 178,181.5Z"/></g><g fill="url(#g-dec)"><g clip-path="url(#clip-d)"><rect x="227.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="227.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="227.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="227.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="237.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="237.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="237.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="237.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="247.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="247.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="247.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="247.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="257.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="257.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="257.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="257.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="267.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="267.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="267.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="267.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="277.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="277.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="277.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="277.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="287.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="287.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="287.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="287.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="297.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="297.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="297.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="297.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="307.5" y="0" width="7.5" height="7.5" rx="1"/><rect x="307.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="307.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="307.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="317.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="317.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="317.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="327.5" y="10" width="7.5" height="7.5" rx="1"/><rect x="327.5" y="20" width="7.5" height="7.5" rx="1"/><rect x="327.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="337.5" y="30" width="7.5" height="7.5" rx="1"/><rect x="227.5" y="40" width="37.5" height="37.5" rx="2"/><rect x="307.5" y="40" width="7.5" height="7.5" rx="1"/><rect x="307.5" y="50" width="7.5" height="7.5" rx="1"/><rect x="307.5" y="60" width="7.5" height="7.5" rx="1"/><rect x="307.5" y="70" width="7.5" height="7.5" rx="1"/><rect x="317.5" y="40" width="7.5" height="7.5" rx="1"/><rect x="317.5" y="50" width="7.5" height="7.5" rx="1"/><rect x="317.5" y="60" width="7.5" height="7.5" rx="1"/><rect x="317.5" y="70" width="7.5" height="7.5" rx="1"/><rect x="327.5" y="40" width="7.5" height="7.5" rx="1"/><rect x="327.5" y="50" width="7.5" height="7.5" rx="1"/><rect x="327.5" y="60" width="7.5" height="7.5" rx="1"/><rect x="327.5" y="70" width="7.5" height="7.5" rx="1"/><rect x="337.5" y="40" width="7.5" height="7.5" rx="1"/><rect x="337.5" y="50" width="7.5" height="7.5" rx="1"/><rect x="337.5" y="60" width="7.5" height="7.5" rx="1"/><rect x="337.5" y="70" width="7.5" height="7.5" rx="1"/><rect x="227.5" y="80" width="37.5" height="37.5" rx="2"/><rect x="307.5" y="80" width="37.5" height="37.5" rx="2"/><rect x="227.5" y="120" width="37.5" height="37.5" rx="2"/><rect x="307.5" y="120" width="37.5" height="37.5" rx="2"/><rect x="227.5" y="160" width="37.5" height="37.5" rx="2"/><rect x="267.5" y="160" width="37.5" height="37.5" rx="2"/><rect x="307.5" y="160" width="37.5" height="37.5" rx="2"/></g><rect x="237.5" y="200" width="17.5" height="22" rx="1.5"/><rect x="277.5" y="200" width="17.5" height="22" rx="1.5"/><path d="M393.25 199.26Q380.53 199.26 372.53 194.505Q364.53 189.75 360.76 180.54000000000002Q356.99 171.33 356.99 158.15V124.46Q356.99 110.99 360.81 101.875Q364.63 92.76 372.725 88.055Q380.82 83.35 393.25 83.35Q406.54 83.35 414.05 88.38499999999999Q421.56 93.42 424.755 103.05000000000001Q427.95 112.68 427.95 126.43V142.5H383.21V164.55Q383.21 170.05 384.325 173.525Q385.44 177.0 387.70500000000004 178.59Q389.97 180.18 393.34 180.18Q396.8 180.18 398.93 178.54500000000002Q401.06 176.91 402.03499999999997 173.685Q403.01 170.46 403.01 165.62V156.21H427.85V163.9Q427.85 180.92 419.25 190.08999999999997Q410.65 199.26 393.25 199.26ZM383.21 129.41H403.01V118.7Q403.01 113.0 402.08500000000004 109.39Q401.16 105.78 399.03 104.105Q396.9 102.43 393.05 102.43Q389.4 102.43 387.28 104.16Q385.16 105.89 384.185 110.025Q383.21 114.16 383.21 121.58ZM481.68 199.26Q468.68 199.26 460.78 194.26999999999998Q452.88 189.28 449.29499999999996 180.07Q445.71 170.86 445.71 158.25V124.46Q445.71 111.37 449.34000000000003 102.21000000000001Q452.97 93.05 460.87 88.19999999999999Q468.77 83.35 481.68 83.35Q493.83 83.35 501.44 87.19Q509.05 91.03 512.585 98.7Q516.12 106.37 516.12 117.94V127.26H491.23V117.35Q491.23 111.46 490.31 108.13499999999999Q489.39 104.81 487.315 103.37Q485.24 101.93 481.78 101.93Q478.13 101.93 476.01 103.71000000000001Q473.89 105.49 473.065 109.625Q472.24 113.76 472.24 120.98V162.19Q472.24 173.07 474.51 176.825Q476.78 180.58 481.97 180.58Q485.82 180.58 487.79499999999996 178.85500000000002Q489.77 177.13 490.5 173.70999999999998Q491.23 170.29 491.23 165.45V153.3H516.12V164.09Q516.12 175.37 512.49 183.285Q508.86 191.2 501.25 195.23Q493.64 199.26 481.68 199.26ZM569.53 199.26Q557.29 199.26 549.24 194.70499999999998Q541.19 190.15 537.2750000000001 181.42000000000002Q533.36 172.69 533.36 160.25V122.36Q533.36 109.92 537.2750000000001 101.19Q541.19 92.46 549.24 87.905Q557.29 83.35 569.53 83.35Q581.78 83.35 589.78 87.905Q597.78 92.46 601.745 101.19Q605.71 109.92 605.71 122.36V160.25Q605.71 172.69 601.745 181.42000000000002Q597.78 190.15 589.78 194.70499999999998Q581.78 199.26 569.53 199.26ZM569.63 180.98Q574.15 180.98 576.23 178.435Q578.31 175.89 578.9 171.56Q579.49 167.23 579.49 162.01V120.69Q579.49 115.38 578.9 111.095Q578.31 106.81 576.23 104.22Q574.15 101.63 569.63 101.63Q565.11 101.63 562.985 104.22Q560.86 106.81 560.22 111.095Q559.58 115.38 559.58 120.69V162.01Q559.58 167.23 560.22 171.56Q560.86 175.89 562.985 178.435Q565.11 180.98 569.63 180.98ZM651.13 199.26Q637.84 199.26 630.94 189.70999999999998Q624.04 180.16 624.04 159.28V124.76Q624.04 113.13 626.5799999999999 103.75Q629.12 94.37 635.0 88.86Q640.88 83.35 650.85 83.35Q656.79 83.35 661.9549999999999 85.83Q667.12 88.31 671.32 92.21V40.0H697.75V197.5H671.32V189.5Q666.82 193.99 661.7950000000001 196.625Q656.77 199.26 651.13 199.26ZM660.79 180.92Q663.01 180.92 665.765 180.0Q668.52 179.08 671.32 177.34V105.05Q669.0 103.61 666.2950000000001 102.5Q663.59 101.39 660.71 101.39Q655.02 101.39 652.79 106.59Q650.56 111.79 650.56 119.1V161.13Q650.56 166.7 651.385 171.21499999999997Q652.21 175.73 654.4300000000001 178.325Q656.65 180.92 660.79 180.92ZM720.71 197.5V85.11H747.13V197.5ZM720.61 67.18V45.31H747.13V67.18ZM770.17 197.5V85.11H796.6V96.29Q802.54 90.54 809.295 86.945Q816.05 83.35 823.81 83.35Q830.72 83.35 834.8199999999999 86.8Q838.92 90.25 840.8299999999999 95.85499999999999Q842.74 101.46 842.74 108.01V197.5H816.41V113.23Q816.41 108.21 814.8599999999999 105.505Q813.31 102.8 808.77 102.8Q806.07 102.8 802.885 104.34Q799.7 105.88 796.6 108.29V197.5ZM897.7 234.6Q886.19 234.6 877.325 232.215Q868.46 229.83 863.495 224.96Q858.53 220.09 858.53 212.51Q858.53 206.68 861.2 202.20499999999998Q863.87 197.73 868.4 194.515Q872.93 191.3 878.3 189.25L889.33 191.01Q887.0 193.05 884.9200000000001 195.18Q882.84 197.31 881.485 199.63Q880.13 201.95 880.13 204.86Q880.13 209.52 884.755 211.845Q889.38 214.17 900.65 214.17Q910.84 214.17 916.085 212.05Q921.33 209.93 921.33 204.69Q921.33 201.59 919.97 199.70499999999998Q918.61 197.82 915.405 196.66Q912.2 195.5 906.38 194.92L880.07 191.6Q871.78 190.53 867.285 185.94Q862.79 181.35 862.79 175.08Q862.79 168.18 866.875 162.815Q870.96 157.45 877.77 151.62L886.87 157.47Q885.11 159.43 883.56 161.845Q882.01 164.26 882.01 166.88Q882.01 169.12 884.2950000000001 170.485Q886.58 171.85 892.04 172.44L914.92 175.09Q928.74 176.56 935.31 183.865Q941.88 191.17 941.88 203.61Q941.88 213.75 937.3050000000001 220.66500000000002Q932.73 227.58 923.0250000000001 231.09Q913.32 234.6 897.7 234.6ZM899.08 161.99Q886.71 161.99 878.56 157.135Q870.41 152.28 866.405 143.485Q862.4 134.69 862.4 122.81Q862.4 109.99 866.595 101.19Q870.79 92.39 878.9849999999999 87.87Q887.18 83.35 899.08 83.35Q910.87 83.35 918.6700000000001 88.44Q926.47 93.53 930.4300000000001 102.42Q934.39 111.31 934.39 123.1Q934.39 134.97 930.675 143.715Q926.96 152.46 919.2 157.22500000000002Q911.44 161.99 899.08 161.99ZM899.08 145.01Q902.56 145.01 904.635 143.655Q906.71 142.3 907.77 139.59Q908.83 136.88 909.27 132.665Q909.71 128.45 909.71 122.81Q909.71 117.08 909.3199999999999 112.86500000000001Q908.93 108.65 907.8199999999999 105.845Q906.71 103.04 904.635 101.64Q902.56 100.24 899.17 100.24Q895.8 100.24 893.625 101.59Q891.45 102.94 890.245 105.69999999999999Q889.04 108.46 888.605 112.725Q888.17 116.99 888.17 122.81Q888.17 128.16 888.605 132.285Q889.04 136.41 890.15 139.26Q891.26 142.11 893.435 143.56Q895.61 145.01 899.08 145.01ZM928.99 103.37 923.23 95.19Q925.49 90.96 930.38 87.22999999999999Q935.27 83.5 942.1 80.38L948.64 94.79Q946.32 95.37 942.155 96.975Q937.99 98.58 934.145 100.38499999999999Q930.3 102.19 928.99 103.37Z"/></g><g fill="#5e8c56"><rect x="961.92" y="130" width="7.5" height="7.5" rx="1"/><rect x="961.92" y="140" width="7.5" height="7.5" rx="1"/><rect x="971.92" y="130" width="7.5" height="7.5" rx="1"/><rect x="971.92" y="140" width="7.5" height="7.5" rx="1"/><rect x="981.92" y="130" width="7.5" height="7.5" rx="1"/><rect x="981.92" y="140" width="7.5" height="7.5" rx="1"/></g><g fill="url(#g-fp4)"><rect x="1003.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1003.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1003.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1003.42" y="70" width="7.5" height="7.5" rx="1"/><rect x="1013.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1013.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1013.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1013.42" y="70" width="7.5" height="7.5" rx="1"/><rect x="1023.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1023.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1023.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1023.42" y="70" width="7.5" height="7.5" rx="1"/><rect x="1033.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1033.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1043.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1043.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1053.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1053.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1063.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1063.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1093.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1093.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1093.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1093.42" y="70" width="7.5" height="7.5" rx="1"/><rect x="1103.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1103.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1103.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1103.42" y="70" width="7.5" height="7.5" rx="1"/><rect x="1113.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1113.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1113.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1113.42" y="70" width="7.5" height="7.5" rx="1"/><rect x="1123.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1123.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1133.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1133.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1143.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1143.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1143.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1153.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1153.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1153.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1153.42" y="70" width="7.5" height="7.5" rx="1"/><rect x="1163.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1163.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1163.42" y="70" width="7.5" height="7.5" rx="1"/><rect x="1173.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1173.42" y="70" width="7.5" height="7.5" rx="1"/><rect x="1223.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1223.42" y="70" width="7.5" height="7.5" rx="1"/><rect x="1233.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1233.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1233.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1233.42" y="70" width="7.5" height="7.5" rx="1"/><rect x="1243.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1243.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1243.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1243.42" y="70" width="7.5" height="7.5" rx="1"/><rect x="1253.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1253.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1253.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1253.42" y="70" width="7.5" height="7.5" rx="1"/><rect x="1263.42" y="40" width="7.5" height="7.5" rx="1"/><rect x="1263.42" y="50" width="7.5" height="7.5" rx="1"/><rect x="1263.42" y="60" width="7.5" height="7.5" rx="1"/><rect x="1263.42" y="70" width="7.5" height="7.5" rx="1"/><g clip-path="url(#clip-fp4)"><rect x="999.42" y="80" width="294.25" height="118.5"/></g></g></svg>diff --git a/pyproject.toml b/pyproject.tomlnew file mode 100644index 0000000..09ff5d3--- /dev/null+++ b/pyproject.toml@@ -0,0 +1,36 @@+[build-system]+requires = ["setuptools>=64"]+build-backend = "setuptools.build_meta"++[project]+name = "bitdecoding-fp4-kernel"+version = "0.1.0"+description = "NVFP4 paged-decode kernel interface"+requires-python = ">=3.10"+dependencies = [+ "torch",+ "nvidia-cutlass-dsl==4.6.0",+ "quack-kernels==0.6.1",+ "cuda-python",+ "typing_extensions",+ "apache-tvm-ffi==0.1.10",+ "torch-c-dlpack-ext",+]++# Points `backend: CUSTOM` at NVFP4Backend in every vLLM process.+[project.entry-points."vllm.general_plugins"]+nvfp4_vllm = "nvfp4_vllm:register"++[project.optional-dependencies]+eval = [+ "flashinfer-python",+ "datasets",+ "pandas",+ "pyarrow",+ "safetensors",+ # RULER's synthetic-data generator.+ "nltk",+]++[tool.setuptools.packages.find]+where = ["src"]diff --git a/src/nvfp4_decode_kernel/__init__.py b/src/nvfp4_decode_kernel/__init__.pynew file mode 100644index 0000000..040870b--- /dev/null+++ b/src/nvfp4_decode_kernel/__init__.py@@ -0,0 +1,21 @@+"""Public API for the NVFP4 paged-decode kernel."""++import os++os.environ.setdefault("CUTE_DSL_ENABLE_TVM_FFI", "1")++from .interface import (+ RESIDUAL_ROW_TILE,+ DecodeTailWrite,+ KernelNotAvailableError,+ fp4_decode,+ split_partial_count,+)++__all__ = [+ "RESIDUAL_ROW_TILE",+ "DecodeTailWrite",+ "KernelNotAvailableError",+ "fp4_decode",+ "split_partial_count",+]diff --git a/src/nvfp4_decode_kernel/_block_scale.py b/src/nvfp4_decode_kernel/_block_scale.pynew file mode 100644index 0000000..d334d9d--- /dev/null+++ b/src/nvfp4_decode_kernel/_block_scale.py@@ -0,0 +1,623 @@+"""The block-scale search both quantizers run, and the conversions it needs.++A group of sixteen gets one E4M3 scale. Dividing its largest magnitude by six+puts that magnitude on the top rung of the E2M1 ladder; dividing by less+sacrifices it to place the rest of the group better. Which trade wins is a+property of the group, so this tries each divisor `scale_mode` names, keeps+the one whose round trip costs least, and stores that one. With a single+divisor the search collapses and the result is byte for byte what the+one-divisor quantizer stored.++Two arithmetic choices must match `reference.py` exactly:++- A candidate is judged by its reconstruction, read back through+ ``cvt.rn.f16x2.e2m1x2`` from the very bytes that would be stored. Nothing+ re-derives the rounding, so nothing can drift from it.+- The squared error accumulates through explicit ``fma.rn.f32``, in element+ order. Written as ``error + difference * difference`` the backend contracts+ it into an FMA anyway; spelling it out fixes the arithmetic against a+ backend that someday would not, and the last place it settles decides which+ candidate wins a group where two are level.+"""++import cutlass+import cutlass.cute as cute+from cutlass import Float32, Int32+from cutlass._mlir.dialects import llvm+from cutlass.cutlass_dsl import T+++GROUPS_PER_ATOM = 4+GAUSSIAN_E2M1_NORM_SCALE = 0.04305219637254294+++def _load_vector(source: cute.Tensor) -> cute.Tensor:+ """Bring one thread's whole vector into registers in one vectorized copy."""+ values = cute.make_fragment_like(source)+ cute.autovec_copy(source, values)+ return values+++def _pack_e4m3(+ f0: Float32,+ f1: Float32,+ f2: Float32,+ f3: Float32,+ *,+ loc=None,+ ip=None,+) -> Int32:+ packed = llvm.inline_asm(+ T.i32(),+ [+ Float32(f0).ir_value(loc=loc, ip=ip),+ Float32(f1).ir_value(loc=loc, ip=ip),+ Float32(f2).ir_value(loc=loc, ip=ip),+ Float32(f3).ir_value(loc=loc, ip=ip),+ ],+ "{\n\t"+ ".reg .b16 lo;\n\t"+ ".reg .b16 hi;\n\t"+ "cvt.rn.satfinite.e4m3x2.f32 lo, $2, $1;\n\t"+ "cvt.rn.satfinite.e4m3x2.f32 hi, $4, $3;\n\t"+ "mov.b32 $0, {lo, hi};\n\t"+ "}\n",+ "=r,f,f,f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ return Int32(packed)+++def _pack_e2m1(+ f0: Float32,+ f1: Float32,+ f2: Float32,+ f3: Float32,+ f4: Float32,+ f5: Float32,+ f6: Float32,+ f7: Float32,+ *,+ loc=None,+ ip=None,+) -> Int32:+ packed = llvm.inline_asm(+ T.i32(),+ [+ Float32(f0).ir_value(loc=loc, ip=ip),+ Float32(f1).ir_value(loc=loc, ip=ip),+ Float32(f2).ir_value(loc=loc, ip=ip),+ Float32(f3).ir_value(loc=loc, ip=ip),+ Float32(f4).ir_value(loc=loc, ip=ip),+ Float32(f5).ir_value(loc=loc, ip=ip),+ Float32(f6).ir_value(loc=loc, ip=ip),+ Float32(f7).ir_value(loc=loc, ip=ip),+ ],+ "{\n\t"+ ".reg .b8 byte0;\n\t"+ ".reg .b8 byte1;\n\t"+ ".reg .b8 byte2;\n\t"+ ".reg .b8 byte3;\n\t"+ "cvt.rn.satfinite.e2m1x2.f32 byte0, $2, $1;\n\t"+ "cvt.rn.satfinite.e2m1x2.f32 byte1, $4, $3;\n\t"+ "cvt.rn.satfinite.e2m1x2.f32 byte2, $6, $5;\n\t"+ "cvt.rn.satfinite.e2m1x2.f32 byte3, $8, $7;\n\t"+ "mov.b32 $0, {byte0, byte1, byte2, byte3};\n\t"+ "}\n",+ "=r,f,f,f,f,f,f,f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ return Int32(packed)+++def _widen_e2m1_byte(+ packed: Int32,+ byte: cutlass.Constexpr[int],+ *,+ loc=None,+ ip=None,+) -> Int32:+ """One byte of two E2M1 nibbles as an FP16 pair, low nibble in the low half."""+ widened = llvm.inline_asm(+ T.i32(),+ [Int32(packed).ir_value(loc=loc, ip=ip)],+ "{\n\t"+ ".reg .b32 shifted;\n\t"+ ".reg .b8 nibbles;\n\t"+ ".reg .b32 pair;\n\t"+ f"shr.b32 shifted, $1, {8 * byte};\n\t"+ "cvt.u8.u32 nibbles, shifted;\n\t"+ "cvt.rn.f16x2.e2m1x2 pair, nibbles;\n\t"+ "mov.b32 $0, pair;\n\t"+ "}\n",+ "=r,r",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ return Int32(widened)+++def _unpack_e2m1(packed: Int32):+ """The eight magnitudes a packed word encodes, in element order.++ FP16 holds every E2M1 value exactly, and so does FP32, so widening twice+ is lossless and what comes back is the grid point the store committed to.+ """+ halves = cute.make_rmem_tensor(cute.make_layout(8), cutlass.Float16)+ words = cute.recast_tensor(halves, cutlass.Int32)+ for byte in cutlass.range_constexpr(4):+ words[byte] = _widen_e2m1_byte(packed, byte)+ magnitudes = [Float32(0.0)] * 8+ for index in cutlass.range_constexpr(8):+ magnitudes[index] = Float32(halves[index])+ return magnitudes+++def _fma(a: Float32, b: Float32, c: Float32, *, loc=None, ip=None) -> Float32:+ """``a * b + c`` with one rounding, whatever the backend would have done."""+ fused = llvm.inline_asm(+ T.f32(),+ [+ Float32(a).ir_value(loc=loc, ip=ip),+ Float32(b).ir_value(loc=loc, ip=ip),+ Float32(c).ir_value(loc=loc, ip=ip),+ ],+ "fma.rn.f32 $0, $1, $2, $3;",+ "=f,f,f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ return Float32(fused)+++def _sqrt(value: Float32, *, loc=None, ip=None) -> Float32:+ """One correctly rounded FP32 square root."""+ result = llvm.inline_asm(+ T.f32(),+ [Float32(value).ir_value(loc=loc, ip=ip)],+ "sqrt.rn.f32 $0, $1;",+ "=f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ return Float32(result)+++def gaussian_global_scale(values: cute.Tensor) -> Float32:+ """One Gaussian-optimal E2M1 scale for a complete 128-channel vector."""+ energy = Float32(0.0)+ for element in cutlass.range_constexpr(128):+ value = Float32(values[element])+ energy = _fma(value, value, energy)+ raw = _sqrt(energy) * Float32(GAUSSIAN_E2M1_NORM_SCALE)+ rounded = cute.make_rmem_tensor(+ cute.make_layout(GROUPS_PER_ATOM), cutlass.Float8E4M3FN+ )+ cute.recast_tensor(rounded, cutlass.Int32)[0] = _pack_e4m3(+ raw, raw, raw, raw+ )+ loaded = rounded.load().to(Float32)+ return Float32(loaded[0])+++def _round_trip(+ source: cute.Tensor,+ start: Int32,+ scale: Float32,+ sf_vec_size: cutlass.Constexpr[int],+):+ """One candidate over one group: the two words it stores and what they cost.++ A scale that flushed to zero throws every nonzero element past the top of+ the grid, where E2M1 saturates at six; multiplying back by the zero scale+ returns the group as zeros, which is what the decode path would read. The+ error then equals the group's own energy, so the candidate loses to any+ that can represent anything at all, and wins only where there is nothing+ to represent.+ """+ inverse = Float32(1.0) / (scale + Float32(1.0e-30))+ scaled = [Float32(0.0)] * 16+ for element in cutlass.range_constexpr(sf_vec_size):+ scaled[element] = Float32(source[start + element]) * inverse++ low = _pack_e2m1(+ scaled[0],+ scaled[1],+ scaled[2],+ scaled[3],+ scaled[4],+ scaled[5],+ scaled[6],+ scaled[7],+ )+ high = _pack_e2m1(+ scaled[8],+ scaled[9],+ scaled[10],+ scaled[11],+ scaled[12],+ scaled[13],+ scaled[14],+ scaled[15],+ )++ magnitudes = _unpack_e2m1(low) + _unpack_e2m1(high)+ error = Float32(0.0)+ for element in cutlass.range_constexpr(sf_vec_size):+ difference = _fma(+ magnitudes[element],+ scale,+ -Float32(source[start + element]),+ )+ error = _fma(difference, difference, error)+ return low, high, error+++def _candidate_scale(maximum: Float32, divisor: cutlass.Constexpr) -> Float32:+ """``maximum / divisor`` as the E4M3 scale it would be stored as.++ The kernel multiplies by a reciprocal constant where a reader might divide,+ and through E4M3's three mantissa bits the two disagree on some amax. The+ conversion is per element, so packing one value four times stores the byte+ that packing four different groups' values would have stored for it.+ """+ scaled = maximum * Float32(1.0 / divisor)+ rounded = cute.make_rmem_tensor(+ cute.make_layout(GROUPS_PER_ATOM), cutlass.Float8E4M3FN+ )+ cute.recast_tensor(rounded, cutlass.Int32)[0] = _pack_e4m3(+ scaled, scaled, scaled, scaled+ )+ loaded = rounded.load().to(Float32)+ return Float32(loaded[0])+++def select_group_scale(+ values: cute.Tensor,+ start,+ sf_vec_size: cutlass.Constexpr[int],+ divisors: cutlass.Constexpr,+):+ """Choose one group's scale and return it with the exact RTN words it won.++ A group's result depends on its own sixteen values and nothing else, which+ is what lets one thread own one group as well as one thread own a vector.++ ``divisors`` is in tie-break order: a candidate replaces the incumbent+ only on a strictly lower error, so the first divisor wins a tie and a+ longer tuple can only improve on a shorter prefix of itself.+ """+ maximum = Float32(0.0)+ for element in cutlass.range_constexpr(sf_vec_size):+ value = Float32(values[start + element])+ maximum = cute.arch.fmax(maximum, cute.arch.fmax(value, -value))++ scale = _candidate_scale(maximum, divisors[0])+ low, high, error = _round_trip(+ values, start, scale, sf_vec_size+ )+ for index in cutlass.range_constexpr(1, len(divisors)):+ rival = _candidate_scale(maximum, divisors[index])+ rival_low, rival_high, rival_error = _round_trip(+ values, start, rival, sf_vec_size+ )+ better = rival_error < error+ scale = Float32(cutlass.select_(better, rival, scale))+ low = Int32(cutlass.select_(better, rival_low, low))+ high = Int32(cutlass.select_(better, rival_high, high))+ error = Float32(cutlass.select_(better, rival_error, error))+ return scale, low, high+++def select_scales(+ values: cute.Tensor,+ k_atom: cutlass.Constexpr[int],+ sf_vec_size: cutlass.Constexpr[int],+ divisors: cutlass.Constexpr,+) -> tuple[cute.Tensor, cute.Tensor]:+ """Choose four scales and return them with the exact RTN words they won.++ The quantizer stores ``words`` directly, preserving the reconstruction+ used to select each scale and the candidate tie order.+ """+ winners = cute.make_rmem_tensor(+ cute.make_layout(GROUPS_PER_ATOM), Float32+ )+ words = cute.make_rmem_tensor(+ cute.make_layout(2 * GROUPS_PER_ATOM), Int32+ )+ for group in cutlass.range_constexpr(GROUPS_PER_ATOM):+ group_index = k_atom * GROUPS_PER_ATOM + group+ scale, low, high = select_group_scale(+ values, group_index * sf_vec_size, sf_vec_size, divisors+ )+ winners[group] = scale+ words[2 * group] = low+ words[2 * group + 1] = high++ return winners, words+++def _store_atom(+ words: cute.Tensor,+ destination: cute.Tensor,+ k_atom: cutlass.Constexpr[int],+ sf_vec_size: cutlass.Constexpr[int],+):+ """Store one atom's packed words (eight for NVFP4, sixteen for MXFP4) in one vector copy."""+ atom_byte_offset = k_atom * GROUPS_PER_ATOM * (sf_vec_size // 2)+ # The validated K/V layouts place every vector, and therefore every+ # atom within it, on a 16-byte boundary. Restoring that alignment+ # after the dynamic slice lets autovec_copy issue one coalesced store.+ atom_pointer = (destination.iterator + atom_byte_offset).align(16)+ atom_destination = cute.make_tensor(+ cute.recast_ptr(atom_pointer, dtype=cutlass.Int32),+ cute.make_layout(GROUPS_PER_ATOM * sf_vec_size // 8),+ )+ cute.autovec_copy(words, atom_destination)+++MX_SF_VEC_SIZE = 32+++def _float_bits(value: Float32, *, loc=None, ip=None) -> Int32:+ bits = llvm.inline_asm(+ T.i32(),+ [Float32(value).ir_value(loc=loc, ip=ip)],+ "mov.b32 $0, $1;",+ "=r,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ return Int32(bits)+++def _bits_float(bits: Int32, *, loc=None, ip=None) -> Float32:+ value = llvm.inline_asm(+ T.f32(),+ [Int32(bits).ir_value(loc=loc, ip=ip)],+ "mov.b32 $0, $1;",+ "=f,r",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ return Float32(value)+++def power_of_two(exponent: Int32) -> Float32:+ """``2**exponent`` for an E8M0 exponent; 2**-127 is the one subnormal."""+ normal = (exponent + Int32(127)) << Int32(23)+ return _bits_float(+ Int32(cutlass.select_(exponent < Int32(-126), Int32(1 << 22), normal))+ )+++def ocp_exponent(amax: Float32) -> Int32:+ """``floor(log2(amax)) - 2`` clamped to E8M0, read from the float's exponent field.++ A zero or subnormal amax has field 0 and lands on the clamp at -127, which+ is where the reference puts every group too small to represent.+ """+ exponent = ((_float_bits(amax) >> Int32(23)) & Int32(0xFF)) - Int32(129)+ return Int32(cutlass.select_(exponent < Int32(-127), Int32(-127), exponent))+++def round_up_exponent(amax: Float32) -> Int32:+ """The smallest E8M0 exponent whose scale keeps ``amax`` at or below six.++ ``amax > 6 * 2**ocp`` holds exactly when the mantissa is above 1.5, and+ adding 0x3FFFFF to the bits carries into the exponent field exactly then,+ so the rule costs an add and a shift on the softmax path. A zero, negative+ zero or subnormal amax lands on the clamp at -127.+ """+ exponent = ((_float_bits(amax) + Int32(0x3FFFFF)) >> Int32(23)) - Int32(129)+ return Int32(cutlass.select_(exponent < Int32(-127), Int32(-127), exponent))+++def inverse_power_of_two(exponent: Int32) -> Float32:+ """``2**-exponent`` for an E8M0 exponent below 127, always a normal float."""+ return _bits_float((Int32(127) - exponent) << Int32(23))+++def _mx_round_trip(+ source: cute.Tensor, start: Int32, exponent: Int32,+ width: cutlass.Constexpr[int] = 32,+):+ """An MX group, or one lane's half, on a shared power-of-two scale."""+ scale = power_of_two(exponent)+ inverse = power_of_two(Int32(0) - exponent)+ words = [Int32(0)] * (width // 8)+ for word in cutlass.range_constexpr(width // 8):+ base = start + 8 * word+ words[word] = _pack_e2m1(+ Float32(source[base]) * inverse,+ Float32(source[base + 1]) * inverse,+ Float32(source[base + 2]) * inverse,+ Float32(source[base + 3]) * inverse,+ Float32(source[base + 4]) * inverse,+ Float32(source[base + 5]) * inverse,+ Float32(source[base + 6]) * inverse,+ Float32(source[base + 7]) * inverse,+ )+ # Match Q's two-lane group: FMA within each half, then one FP32 sum.+ errors = [Float32(0.0)] * (width // 16)+ for half in cutlass.range_constexpr(width // 16):+ for word in cutlass.range_constexpr(2):+ index = 2 * half + word+ magnitudes = _unpack_e2m1(words[index])+ for element in cutlass.range_constexpr(8):+ difference = _fma(+ magnitudes[element],+ scale,+ -Float32(source[start + 8 * index + element]),+ )+ errors[half] = _fma(difference, difference, errors[half])+ error = errors[0]+ if cutlass.const_expr(width == 32):+ error = error + errors[1]+ return words, error+++def mx_quantize_atom(+ values: cute.Tensor,+ destination: cute.Tensor,+ k_atom: cutlass.Constexpr[int],+ rule: cutlass.Constexpr[str],+ residual: cute.Tensor = None,+) -> Int32:+ """Quantize four 32-value groups to MXFP4 into ``destination``; return their E8M0 bytes.++ ``rule`` is ``ocp``, ``round_up`` or ``search``, decided exactly as+ ``reference.mxfp4_quantize`` decides it: ``round_up`` takes the doubled+ exponent when the group maximum exceeds six OCP scales+ (:func:`round_up_exponent`), ``search`` when the doubled exponent's squared+ error is strictly lower. With ``residual`` it also writes the exact FP32+ reconstruction error from the stored words and scale.+ """+ words = cute.make_rmem_tensor(+ cute.make_layout(GROUPS_PER_ATOM * MX_SF_VEC_SIZE // 8), Int32+ )+ packed = Int32(0)+ for group in cutlass.range_constexpr(GROUPS_PER_ATOM):+ start = (k_atom * GROUPS_PER_ATOM + group) * MX_SF_VEC_SIZE+ maximum = Float32(0.0)+ for element in cutlass.range_constexpr(MX_SF_VEC_SIZE):+ value = Float32(values[start + element])+ maximum = cute.arch.fmax(maximum, cute.arch.fmax(value, -value))+ if cutlass.const_expr(rule == "round_up"):+ exponent = round_up_exponent(maximum)+ else:+ exponent = ocp_exponent(maximum)+ group_words, error = _mx_round_trip(values, start, exponent)+ if cutlass.const_expr(rule == "search"):+ doubled = Int32(+ cutlass.select_(exponent < Int32(127), exponent + Int32(1), exponent)+ )+ doubled_words, doubled_error = _mx_round_trip(values, start, doubled)+ take = doubled_error < error+ exponent = Int32(cutlass.select_(take, doubled, exponent))+ for word in cutlass.range_constexpr(4):+ group_words[word] = Int32(+ cutlass.select_(take, doubled_words[word], group_words[word])+ )+ for word in cutlass.range_constexpr(4):+ words[4 * group + word] = group_words[word]+ if cutlass.const_expr(residual is not None):+ scale = power_of_two(exponent)+ for word in cutlass.range_constexpr(4):+ magnitudes = _unpack_e2m1(group_words[word])+ for element in cutlass.range_constexpr(8):+ index = start + 8 * word + element+ residual[index] = _fma(+ -magnitudes[element], scale, Float32(values[index])+ )+ packed = packed | ((exponent + Int32(127)) << Int32(8 * group))+ _store_atom(words, destination, k_atom, MX_SF_VEC_SIZE)+ return packed+++def quantize_atom_with_scale(+ values: cute.Tensor,+ destination: cute.Tensor,+ k_atom: cutlass.Constexpr[int],+ sf_vec_size: cutlass.Constexpr[int],+ scale: Float32,+) -> Int32:+ """Quantize four groups with one already-rounded shared E4M3 scale."""+ words = cute.make_rmem_tensor(+ cute.make_layout(2 * GROUPS_PER_ATOM), Int32+ )+ for group in cutlass.range_constexpr(GROUPS_PER_ATOM):+ start = (k_atom * GROUPS_PER_ATOM + group) * sf_vec_size+ low, high, _ = _round_trip(+ values, start, scale, sf_vec_size+ )+ words[2 * group] = low+ words[2 * group + 1] = high+ _store_atom(words, destination, k_atom, sf_vec_size)+ return _pack_e4m3(scale, scale, scale, scale)+++def quantize_atom(+ values: cute.Tensor,+ destination: cute.Tensor,+ k_atom: cutlass.Constexpr[int],+ sf_vec_size: cutlass.Constexpr[int],+ scale_rule: cutlass.Constexpr,+) -> Int32:+ """Quantize four groups into ``destination``; return their packed scales.++ ``scale_rule`` is ``scale_mode.scale_rule``: NVFP4 divisors, or an MX+ exponent rule, which hands the atom to :func:`mx_quantize_atom`.++ The caller stores the returned word wherever its scale layout puts this+ atom. It is packed last rather than first because which scale each group+ keeps is not known until every candidate has been priced.+ """+ if cutlass.const_expr(isinstance(scale_rule, str)):+ assert sf_vec_size == MX_SF_VEC_SIZE+ return mx_quantize_atom(values, destination, k_atom, scale_rule)+ divisors = scale_rule+ winners, words = select_scales(values, k_atom, sf_vec_size, divisors)+ _store_atom(words, destination, k_atom, sf_vec_size)++ # Re-rounding a value that came out of E4M3 returns it, so this reproduces+ # the byte the winning candidate was judged on rather than approaching it+ # a second time.+ return _pack_e4m3(winners[0], winners[1], winners[2], winners[3])+++def group_residual(+ values: cute.Tensor,+ residual: cute.Tensor,+ start,+ scale: Float32,+ low: Int32,+ high: Int32,+ sf_vec_size: cutlass.Constexpr[int],+):+ """Write ``values - decode(low, high) * scale`` for one group, exactly.++ An E2M1 magnitude carries two significant bits and an E4M3 scale four, so+ their product is exact in FP32 and the FMA rounds only the difference.+ """+ magnitudes = _unpack_e2m1(low) + _unpack_e2m1(high)+ for element in cutlass.range_constexpr(sf_vec_size):+ residual[start + element] = _fma(+ -magnitudes[element],+ scale,+ Float32(values[start + element]),+ )diff --git a/src/nvfp4_decode_kernel/_decode.py b/src/nvfp4_decode_kernel/_decode.pynew file mode 100644index 0000000..e6a3085--- /dev/null+++ b/src/nvfp4_decode_kernel/_decode.py@@ -0,0 +1,1456 @@+"""Narrow host dispatch for the extracted SM100 FP4 paged-decode kernel."""++from __future__ import annotations++import functools+import math+import os+from typing import Any++import cuda.bindings.driver as cuda+import torch++import cutlass+import cutlass.cute as cute+from cutlass import Float32, Int32+from cutlass.cute.runtime import from_dlpack, make_ptr++from .fp4_decode_kernel import FP4DecodeKernel+from .interface import SINK_TOKENS, split_partial_count+from .p_scale import p_scale_is_amax2_or_amax6, resolve_p_scale_environment+++PAGE_SIZE = 128+HEAD_DIM = 128+# Blackwell datacenter major version. The check is on the major alone because+# CuTeDSL targets the live device -- 10.3 compiles to sm_103a, not to sm_100a+# code that would fail to load -- and the tcgen05 atoms this kernel builds on+# are common to the family. Matches NVFP4Backend.supports_compute_capability,+# which has always admitted the family; the two disagreeing meant an engine+# that started and then died on its first decode.+SM100_MAJOR = 10+# Request the transposed softmax layout. The kernel narrows this to the shapes+# it supports; everything else falls back to the untransposed path.+_TRANSPOSE_S = True+++_P_SCALE_MODE, _P_SCALE_DIVISOR, _P_SCALE_GRID_SEARCH = (+ resolve_p_scale_environment()+)+_P_SCALE_TWO_LEVEL = _P_SCALE_MODE == "two_level"+_P_SCALE_RESCALE_LOG2 = 3 if _P_SCALE_MODE == "amax2_or_amax6_c8" else 0+_P_SCALE_AMAX2_OR_AMAX6 = p_scale_is_amax2_or_amax6(_P_SCALE_MODE)++# Per scale format: the kernel's scale type, values per scale, and the torch+# dtype its scale bytes are viewed as.+SCALE_FORMATS = {+ "nvfp4": (cutlass.Float8E4M3FN, 16, torch.float8_e4m3fn),+ "mxfp4": (cutlass.Float8E8M0FNU, 32, torch.float8_e8m0fnu),+}+_decode_compile_cache: dict[tuple[Any, ...], Any] = {}+_split_decode_compile_cache: dict[tuple[Any, ...], Any] = {}+++def split_k_heuristic(+ rows: int,+ heads_kv: int,+ max_pages_per_row: int,+ *,+ sms: int,+) -> int:+ """Select a fixed split count for pure FP4 decode by predicted makespan.++ Every CTA runs one row-head pair over its share of the page blocks, so the+ time the launch takes is the number of scheduling rounds times the blocks+ one CTA walks in a round. Splitting divides the second factor and multiplies+ the first, which is why it is not monotonically good: at 128 unsplit CTAs on+ 148 SMs, two splits turn one round of 128 blocks into two rounds of 64 and+ the makespan does not move at all, while the split path still pays for its+ capped pipeline depth, its FP32 partials, its lost persistent scheduler and+ a second launch. Only accept a split count that beats the unsplit makespan+ by more than that overhead, and among equals take the fewest splits.++ Power-of-two candidates keep one compiled kernel per useful occupancy tier+ and match the combine kernel's fixed workspace contract.++ ``max_pages_per_row`` is the page table's second extent, not the pages the+ rows actually consume, because the real lengths live in ``seqused_fp4`` on+ the device and reading them here would force a synchronization on every+ decode. Callers must therefore size the page table to the batch's own+ longest row; a table left at some global capacity makes short batches look+ long.+ """+ if rows < 1 or heads_kv < 1 or max_pages_per_row < 2 or sms < 1:+ return 1+ unsplit_ctas = rows * heads_kv++ # Filling and draining the KV pipeline costs a CTA roughly this many block+ # times on top of the blocks it actually walks, and it is paid once per+ # scheduling round. Without it the model would keep splitting until every+ # CTA held one block, because it would see shortening the main loop as free.+ prologue_blocks = 8++ def makespan(splits: int) -> int:+ rounds = math.ceil(unsplit_ctas * splits / sms)+ return rounds * (math.ceil(max_pages_per_row / splits) + prologue_blocks)++ # Residual cost of the split path once its depth cap is lifted, as a+ # fraction so the comparison stays in integer arithmetic.+ overhead_num, overhead_den = 6, 5+ # A split whose main loop is a single page block is all prologue and combine.+ min_pages_per_split = 2++ best_splits, best_cost = 1, None+ for splits in (2, 4, 8, 16, 32):+ if splits * min_pages_per_split > max_pages_per_row:+ break+ cost = makespan(splits)+ if best_cost is None or cost < best_cost:+ best_splits, best_cost = splits, cost+ if best_cost is None or best_cost * overhead_num >= makespan(1) * overhead_den:+ return 1+ return best_splits++[email protected]_cache(maxsize=None)+def _multiprocessor_count(device_index: int) -> int:+ return torch.cuda.get_device_properties(device_index).multi_processor_count+++def _to_cute_tensor(+ tensor: torch.Tensor,+ *,+ assumed_align: int,+ leading_dim: int,+) -> cute.Tensor:+ result = from_dlpack(+ tensor.detach(),+ assumed_align=assumed_align,+ enable_tvm_ffi=True,+ )+ return result.mark_layout_dynamic(leading_dim=leading_dim)+++def _require_cuda_tensor(+ tensor: torch.Tensor,+ name: str,+ *,+ device: torch.device,+) -> None:+ if not tensor.is_cuda:+ raise ValueError(f"{name} must be a CUDA tensor")+ if tensor.device != device:+ raise ValueError(f"{name} must be on {device}, got {tensor.device}")+++def _scale_format(scale_format: str) -> tuple[Any, int, torch.dtype]:+ try:+ return SCALE_FORMATS[scale_format]+ except KeyError:+ raise ValueError(+ f"scale_format must be one of {sorted(SCALE_FORMATS)}, got {scale_format!r}"+ ) from None+++def _as_scale_dtype(scales: torch.Tensor, name: str, scale_format: str) -> torch.Tensor:+ dtype = _scale_format(scale_format)[2]+ if scales.dtype is dtype:+ return scales+ if scales.dtype is torch.uint8:+ return scales.view(dtype)+ raise ValueError(f"{name} must contain {scale_format} scale-factor bytes")+++def _as_scale_bytes(scales: torch.Tensor, name: str, scale_format: str) -> torch.Tensor:+ if scales.dtype is torch.uint8:+ return scales+ if scales.dtype is _scale_format(scale_format)[2]:+ return scales.view(torch.uint8)+ raise ValueError(f"{name} must contain {scale_format} scale-factor bytes")+++def _p_scale_options(scale_format: str) -> tuple[float, bool, bool, bool, int]:+ """The P scale the kernel is built with. MXFP4 has one rule, so the NVFP4+ P-scale environment overrides are refused rather than ignored."""+ if scale_format == "mxfp4":+ if any(name in os.environ for name in+ ("NVFP4_P_SCALE_MODE", "NVFP4_P_SCALE_DIVISOR")):+ raise ValueError("the NVFP4 P-scale overrides do not apply to MXFP4")+ return 6.0, False, False, False, 0+ return (_P_SCALE_DIVISOR, _P_SCALE_GRID_SEARCH, _P_SCALE_AMAX2_OR_AMAX6,+ _P_SCALE_TWO_LEVEL, _P_SCALE_RESCALE_LOG2)+++def _page_stride_bytes(pages: torch.Tensor, name: str) -> int:+ """Validate a paged tensor's strides and return its page pitch in bytes.++ Everything below the page axis must be densely packed, because that is the+ shape the quantizer writes and the MMA reads. The page axis itself is free:+ the quantizer's own output packs pages back to back, while a page of a vLLM+ cache is one region of a wider page that also carries the scale factors and+ the other of K/V. TMA only requires that every stride above the contiguous+ axis be a whole number of 16-byte lines.+ """+ itemsize = pages.element_size()+ dense_stride = 1+ for axis in range(pages.ndim - 1, 0, -1):+ if pages.stride(axis) != dense_stride:+ raise ValueError(+ f"{name} must be densely packed within a page; axis {axis} has "+ f"stride {pages.stride(axis)}, expected {dense_stride}"+ )+ dense_stride *= pages.shape[axis]+ page_stride = pages.stride(0)+ if page_stride < dense_stride or (page_stride * itemsize) % 16:+ raise ValueError(+ f"{name} needs a page stride of at least {dense_stride} elements, "+ f"aligned to 16 bytes, got {page_stride}"+ )+ return page_stride * itemsize+++def _page_scales_for_kernel(+ scales: torch.Tensor,+ name: str,+) -> torch.Tensor:+ """Expose a quantizer scale slab in the kernel's seven-axis layout."""+ pages, _, _, rest_m, _, rest_k, heads = scales.shape+ atom_bytes = 512+ m_block_stride = rest_k * atom_bytes+ head_stride = rest_m * m_block_stride+ dense_page_stride = heads * head_stride+ # Only the swizzle below the page axis is fixed. The page pitch comes from+ # the caller so that a scale region carved out of a paged cache works, and+ # as_strided keeps that region's storage offset.+ page_stride = scales.stride()[0]+ expected_strides = (+ page_stride,+ 16,+ 4,+ m_block_stride,+ 1,+ atom_bytes,+ head_stride,+ )+ if scales.stride() != expected_strides:+ raise ValueError(+ f"{name} must use the scale layout returned by the K/V quantizer"+ )+ # 512 is the scale-factor atom the SFB TMA descriptor is built around, and+ # the kernel is told the page pitch keeps that alignment.+ if page_stride < dense_page_stride or page_stride % 512:+ raise ValueError(+ f"{name} needs a page stride of at least {dense_page_stride} "+ f"bytes, aligned to 512, got {page_stride}"+ )+ return scales.as_strided(+ (32, 4, rest_m, 4, rest_k, heads, pages),+ (+ 16,+ 4,+ m_block_stride,+ 1,+ atom_bytes,+ head_stride,+ page_stride,+ ),+ )+++def _check_device_values(condition: torch.Tensor, message: str) -> None:+ if bool(torch.any(condition).item()):+ raise ValueError(message)+++def _split_workspace(+ scratch: torch.Tensor | None,+ name: str,+ shape: tuple[int, ...],+ device: torch.device,+) -> torch.Tensor:+ """A split-K partial buffer of ``shape``, viewed out of the caller's.++ The view always starts at the buffer's first element, so one buffer sized+ for the widest split launch serves every narrower one at the same+ address, which is what a captured graph replays.+ """+ elements = math.prod(shape)+ if scratch is None:+ return torch.empty(shape, dtype=torch.float32, device=device)+ if (+ scratch.dtype is not torch.float32+ or scratch.device != device+ or scratch.ndim != 1+ or not scratch.is_contiguous()+ or scratch.storage_offset() != 0+ or scratch.numel() < elements+ ):+ raise ValueError(+ f"{name} must be a contiguous 1-D FP32 tensor on {device}, "+ f"starting at storage offset zero, with at least {elements} "+ "elements"+ )+ return scratch[:elements].view(shape)+++def _compile_decode(+ *,+ query_fp4: torch.Tensor,+ query_scales: torch.Tensor,+ query_fp4_second: torch.Tensor | None,+ query_scales_second: torch.Tensor | None,+ query_padded_bf16: torch.Tensor | None,+ key_pages_fp4: torch.Tensor,+ key_scales: torch.Tensor,+ value_pages_fp4: torch.Tensor,+ value_scales: torch.Tensor,+ fp4_page_table: torch.Tensor,+ seqused_fp4: torch.Tensor,+ output: torch.Tensor,+ softmax_scale: float,+ residual_key_pages_bf16: torch.Tensor | None,+ residual_value_pages_bf16: torch.Tensor | None,+ residual_page_ids: torch.Tensor | None,+ seqused_residual: torch.Tensor | None,+ seqused_sink: torch.Tensor | None,+ out_indices: torch.Tensor | None,+ scale_format: str,+) -> Any:+ heads_q = query_fp4.shape[2]+ heads_kv = key_pages_fp4.shape[2]+ qhead_per_kvhead = heads_q // heads_kv+ sf_dtype, sf_vec_size, _ = _scale_format(scale_format)+ p_divisor, p_grid_search, p_amax2_or_amax6, p_two_level, p_rescale_log2 = _p_scale_options(scale_format)+ has_residual = residual_key_pages_bf16 is not None+ has_sink = seqused_sink is not None+ query_terms = 2 if query_fp4_second is not None else 1+ cache_key = (+ query_fp4.device.index,+ heads_q,+ heads_kv,+ has_residual,+ has_sink,+ out_indices is not None,+ query_terms,+ scale_format,+ p_divisor,+ p_grid_search,+ p_amax2_or_amax6,+ p_two_level,+ p_rescale_log2,+ )+ compiled = _decode_compile_cache.get(cache_key)+ if compiled is not None:+ return compiled++ operation = FP4DecodeKernel(+ HEAD_DIM,+ HEAD_DIM,+ qhead_per_kvhead=qhead_per_kvhead,+ is_causal=False,+ is_local=False,+ is_split_kv=False,+ pack_gqa=True,+ m_block_size=PAGE_SIZE,+ n_block_size=PAGE_SIZE,+ is_persistent=not has_residual,+ score_mod=None,+ mask_mod=None,+ has_aux_tensors=False,+ paged_kv_non_tma=False,+ is_varlen_q=False,+ sf_dtype=sf_dtype,+ sf_vec_size=sf_vec_size,+ fused_residual_first_block=has_residual,+ fused_sink_block=has_sink,+ residual_source="paged_bf16",+ use_out_indices=out_indices is not None,+ seqlen_q_static_one=True,+ transpose_s=_TRANSPOSE_S,+ query_terms=query_terms,+ p_scale_divisor=p_divisor,+ p_scale_grid_search=p_grid_search,+ p_scale_amax2_or_amax6=p_amax2_or_amax6,+ p_scale_two_level=p_two_level,+ p_scale_rescale_log2=p_rescale_log2,+ )+ fake_stream = cute.runtime.make_fake_stream()+ q_pointer = make_ptr(+ cutlass.Float4E2M1FN,+ 0,+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ k_pointer = make_ptr(+ cutlass.Float4E2M1FN,+ 0,+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ v_pointer = make_ptr(+ cutlass.Float4E2M1FN,+ 0,+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ output_tensor = _to_cute_tensor(+ output, assumed_align=16, leading_dim=3+ )+ page_table_tensor = _to_cute_tensor(+ fp4_page_table, assumed_align=4, leading_dim=1+ )+ seqused_fp4_tensor = _to_cute_tensor(+ seqused_fp4, assumed_align=4, leading_dim=0+ )+ query_scales_tensor = _to_cute_tensor(+ query_scales, assumed_align=16, leading_dim=3+ )+ query_scales_second_tensor = _to_cute_tensor(+ (+ query_scales_second+ if query_scales_second is not None+ else query_scales+ ),+ assumed_align=16,+ leading_dim=3,+ )+ key_scales_tensor = _to_cute_tensor(+ key_scales, assumed_align=16, leading_dim=3+ )+ value_scales_tensor = _to_cute_tensor(+ value_scales, assumed_align=16, leading_dim=3+ )+ symbolic_q_shape = tuple(Int32(0) for _ in query_fp4.shape)+ symbolic_k_shape = tuple(Int32(0) for _ in key_pages_fp4.shape)+ symbolic_v_shape = (+ Int32(0),+ Int32(0),+ Int32(0),+ Int32(0),+ )+ compile_args = (+ operation,+ q_pointer,+ k_pointer,+ v_pointer,+ output_tensor,+ None,+ Float32(softmax_scale),+ fake_stream,+ None,+ None,+ None,+ seqused_fp4_tensor,+ page_table_tensor,+ None,+ None,+ None,+ None,+ None,+ query_scales_tensor,+ key_scales_tensor,+ value_scales_tensor,+ symbolic_q_shape,+ symbolic_k_shape,+ symbolic_v_shape,+ )+ compile_kwargs: dict[str, Any] = {+ # Always dynamic, so one compiled kernel serves both a densely packed+ # page array and a page carved out of a wider cache page.+ "k_page_stride": Int32(0),+ "v_page_stride": Int32(0),+ "k_sf_page_stride": Int32(0),+ "v_sf_page_stride": Int32(0),+ "mOutIndices": (+ _to_cute_tensor(+ out_indices,+ assumed_align=4,+ leading_dim=0,+ )+ if out_indices is not None+ else None+ ),+ "mQSecond": q_pointer,+ "mSFQSecond": query_scales_second_tensor,+ }+ if has_residual:+ assert residual_key_pages_bf16 is not None+ assert residual_value_pages_bf16 is not None+ assert residual_page_ids is not None+ assert seqused_residual is not None+ compile_kwargs.update(+ mResidualQ=_to_cute_tensor(+ query_padded_bf16,+ assumed_align=16,+ leading_dim=3,+ ),+ mResidualK=None,+ mResidualV=None,+ mResidualSeqUsedK=_to_cute_tensor(+ seqused_residual,+ assumed_align=4,+ leading_dim=0,+ ),+ mResidualKCache=_to_cute_tensor(+ residual_key_pages_bf16,+ assumed_align=16,+ leading_dim=3,+ ),+ mResidualVCache=_to_cute_tensor(+ residual_value_pages_bf16,+ assumed_align=16,+ leading_dim=3,+ ),+ mResidualBlockIds=_to_cute_tensor(+ residual_page_ids,+ assumed_align=4,+ leading_dim=0,+ ),+ mSinkSeqUsedK=(+ _to_cute_tensor(+ seqused_sink,+ assumed_align=4,+ leading_dim=0,+ )+ if seqused_sink is not None+ else None+ ),+ )+ compiled = cute.compile(+ *compile_args,+ **compile_kwargs,+ options="--enable-tvm-ffi",+ )+ _decode_compile_cache[cache_key] = compiled+ return compiled+++def _compile_split_decode(+ *,+ query_fp4: torch.Tensor,+ query_scales: torch.Tensor,+ key_pages_fp4: torch.Tensor,+ key_scales: torch.Tensor,+ value_pages_fp4: torch.Tensor,+ value_scales: torch.Tensor,+ fp4_page_table: torch.Tensor,+ seqused_fp4: torch.Tensor,+ output_partial: torch.Tensor,+ lse_partial: torch.Tensor,+ softmax_scale: float,+ num_splits: int,+ bf16_split: bool = False,+ query_scales_second: torch.Tensor | None = None,+ query_padded_bf16: torch.Tensor | None = None,+ residual_key_pages_bf16: torch.Tensor | None = None,+ residual_value_pages_bf16: torch.Tensor | None = None,+ residual_page_ids: torch.Tensor | None = None,+ seqused_residual: torch.Tensor | None = None,+ seqused_sink: torch.Tensor | None = None,+ scale_format: str = "nvfp4",+) -> Any:+ heads_q = query_fp4.shape[2]+ heads_kv = key_pages_fp4.shape[2]+ has_residual = residual_key_pages_bf16 is not None+ has_sink = seqused_sink is not None+ query_terms = 2 if query_scales_second is not None else 1+ sf_dtype, sf_vec_size, _ = _scale_format(scale_format)+ p_divisor, p_grid_search, p_amax2_or_amax6, p_two_level, p_rescale_log2 = _p_scale_options(scale_format)+ cache_key = (+ query_fp4.device.index,+ heads_q,+ heads_kv,+ num_splits,+ has_residual,+ has_sink,+ bf16_split,+ query_terms,+ scale_format,+ p_divisor,+ p_grid_search,+ p_amax2_or_amax6,+ p_two_level,+ p_rescale_log2,+ )+ compiled = _split_decode_compile_cache.get(cache_key)+ if compiled is not None:+ return compiled++ operation = FP4DecodeKernel(+ HEAD_DIM,+ HEAD_DIM,+ qhead_per_kvhead=heads_q // heads_kv,+ is_causal=False,+ is_local=False,+ is_split_kv=True,+ pack_gqa=True,+ m_block_size=PAGE_SIZE,+ n_block_size=PAGE_SIZE,+ is_persistent=False,+ score_mod=None,+ mask_mod=None,+ has_aux_tensors=False,+ paged_kv_non_tma=False,+ is_varlen_q=False,+ sf_dtype=sf_dtype,+ sf_vec_size=sf_vec_size,+ fused_residual_first_block=has_residual,+ fused_sink_block=has_sink,+ bf16_split=bf16_split,+ residual_source="paged_bf16",+ use_out_indices=False,+ seqlen_q_static_one=True,+ transpose_s=_TRANSPOSE_S,+ query_terms=query_terms,+ p_scale_divisor=p_divisor,+ p_scale_grid_search=p_grid_search,+ p_scale_amax2_or_amax6=p_amax2_or_amax6,+ p_scale_two_level=p_two_level,+ p_scale_rescale_log2=p_rescale_log2,+ )+ fake_stream = cute.runtime.make_fake_stream()+ fp4_pointer = lambda: make_ptr(+ cutlass.Float4E2M1FN,+ 0,+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ compile_args = (+ operation,+ fp4_pointer(),+ fp4_pointer(),+ fp4_pointer(),+ _to_cute_tensor(output_partial, assumed_align=16, leading_dim=4),+ _to_cute_tensor(lse_partial, assumed_align=4, leading_dim=3),+ Float32(softmax_scale),+ fake_stream,+ None,+ None,+ None,+ _to_cute_tensor(seqused_fp4, assumed_align=4, leading_dim=0),+ _to_cute_tensor(fp4_page_table, assumed_align=4, leading_dim=1),+ None,+ None,+ None,+ None,+ None,+ _to_cute_tensor(query_scales, assumed_align=16, leading_dim=3),+ _to_cute_tensor(key_scales, assumed_align=16, leading_dim=3),+ _to_cute_tensor(value_scales, assumed_align=16, leading_dim=3),+ tuple(Int32(0) for _ in query_fp4.shape),+ tuple(Int32(0) for _ in key_pages_fp4.shape),+ tuple(Int32(0) for _ in value_pages_fp4.shape),+ )+ compile_kwargs: dict[str, Any] = {+ "k_page_stride": Int32(0),+ "v_page_stride": Int32(0),+ "k_sf_page_stride": Int32(0),+ "v_sf_page_stride": Int32(0),+ "mOutIndices": None,+ "mQSecond": fp4_pointer(),+ "mSFQSecond": _to_cute_tensor(+ query_scales_second+ if query_scales_second is not None+ else query_scales,+ assumed_align=16,+ leading_dim=3,+ ),+ }+ if has_residual:+ assert residual_value_pages_bf16 is not None+ assert residual_page_ids is not None+ assert seqused_residual is not None+ assert query_padded_bf16 is not None+ compile_kwargs.update(+ mResidualQ=_to_cute_tensor(+ query_padded_bf16, assumed_align=16, leading_dim=3+ ),+ mResidualK=None,+ mResidualV=None,+ mResidualSeqUsedK=_to_cute_tensor(+ seqused_residual, assumed_align=4, leading_dim=0+ ),+ mResidualKCache=_to_cute_tensor(+ residual_key_pages_bf16, assumed_align=16, leading_dim=3+ ),+ mResidualVCache=_to_cute_tensor(+ residual_value_pages_bf16, assumed_align=16, leading_dim=3+ ),+ mResidualBlockIds=_to_cute_tensor(+ residual_page_ids, assumed_align=4, leading_dim=0+ ),+ mSinkSeqUsedK=(+ _to_cute_tensor(seqused_sink, assumed_align=4, leading_dim=0)+ if has_sink+ else None+ ),+ )+ compiled = cute.compile(+ *compile_args,+ **compile_kwargs,+ options="--enable-tvm-ffi",+ )+ _split_decode_compile_cache[cache_key] = compiled+ return compiled+++def decode_fp4(+ *,+ query_fp4: torch.Tensor,+ query_scales: torch.Tensor,+ query_fp4_second: torch.Tensor | None = None,+ query_scales_second: torch.Tensor | None = None,+ query_padded_bf16: torch.Tensor | None,+ key_pages_fp4: torch.Tensor,+ key_scales: torch.Tensor,+ value_pages_fp4: torch.Tensor,+ value_scales: torch.Tensor,+ fp4_page_table: torch.Tensor,+ seqused_fp4: torch.Tensor,+ residual_key_pages_bf16: torch.Tensor | None = None,+ residual_value_pages_bf16: torch.Tensor | None = None,+ residual_page_ids: torch.Tensor | None = None,+ seqused_residual: torch.Tensor | None = None,+ seqused_sink: torch.Tensor | None = None,+ has_bf16: torch.Tensor | None = None,+ softmax_scale: float,+ out: torch.Tensor | None = None,+ out_indices: torch.Tensor | None = None,+ trusted_metadata: bool = False,+ validate_only: bool = False,+ scale_format: str = "nvfp4",+) -> torch.Tensor | None:+ """Validate, compile/cache, and launch FP4 paged decode on SM100.++ ``scale_format`` is ``"nvfp4"`` (E4M3 per 16) or ``"mxfp4"`` (E8M0 per 32),+ and applies to Q, K, V and P alike.+ """+ _, sf_vec_size, _ = _scale_format(scale_format)+ scale_atoms = HEAD_DIM // (4 * sf_vec_size)+ if not torch.cuda.is_available():+ raise RuntimeError("decode_fp4 requires CUDA")+ if not math.isfinite(softmax_scale) or softmax_scale <= 0.0:+ raise ValueError("softmax_scale must be finite and positive")+ if query_fp4.ndim != 4:+ raise ValueError(+ "query_fp4 must have shape [rows, 1, heads_q, 64]"+ )+ device = query_fp4.device+ _require_cuda_tensor(query_fp4, "query_fp4", device=device)+ if torch.cuda.get_device_capability(device)[0] != SM100_MAJOR:+ raise RuntimeError("decode_fp4 requires an SM100-family CUDA device")+ fp4_dtype = getattr(torch, "float4_e2m1fn_x2", None)+ if query_fp4.dtype is not fp4_dtype or not query_fp4.is_contiguous():+ raise ValueError("query_fp4 must be contiguous packed E2M1 FP4")+ if (query_fp4_second is None) != (query_scales_second is None):+ raise ValueError(+ "query_fp4_second and query_scales_second must be provided together"+ )+ if query_fp4_second is not None:+ _require_cuda_tensor(+ query_fp4_second, "query_fp4_second", device=device+ )+ if (+ query_fp4_second.dtype is not fp4_dtype+ or not query_fp4_second.is_contiguous()+ or tuple(query_fp4_second.shape) != tuple(query_fp4.shape)+ ):+ raise ValueError(+ "query_fp4_second must match query_fp4 as contiguous packed "+ "E2M1 FP4"+ )++ rows, query_length, heads_q, packed_head_dim = query_fp4.shape+ if (+ rows < 1+ or query_length != 1+ or packed_head_dim * 2 != HEAD_DIM+ ):+ raise ValueError(+ "query_fp4 must have shape [rows, 1, heads_q, 64]"+ )+ expected_padded_shape = (rows, PAGE_SIZE, heads_q, HEAD_DIM)++ for tensor, name in (+ (key_pages_fp4, "key_pages_fp4"),+ (value_pages_fp4, "value_pages_fp4"),+ (query_scales, "query_scales"),+ (key_scales, "key_scales"),+ (value_scales, "value_scales"),+ (fp4_page_table, "fp4_page_table"),+ (seqused_fp4, "seqused_fp4"),+ ):+ _require_cuda_tensor(tensor, name, device=device)++ packed_types = (torch.uint8, fp4_dtype)+ if (+ key_pages_fp4.dtype not in packed_types+ or key_pages_fp4.ndim != 4+ or key_pages_fp4.shape[1] != PAGE_SIZE+ or key_pages_fp4.shape[3] * 2 != HEAD_DIM+ ):+ raise ValueError(+ "key_pages_fp4 must be packed FP4 with shape "+ "[pages, 128, heads_kv, 64]"+ )+ page_count, _, heads_kv, _ = key_pages_fp4.shape+ if (+ value_pages_fp4.dtype not in packed_types+ or tuple(value_pages_fp4.shape)+ != (page_count, heads_kv, HEAD_DIM, PAGE_SIZE // 2)+ ):+ raise ValueError(+ "value_pages_fp4 must be packed FP4 with shape "+ "[pages, heads_kv, 128, 64]"+ )+ _page_stride_bytes(key_pages_fp4, "key_pages_fp4")+ _page_stride_bytes(value_pages_fp4, "value_pages_fp4")+ if heads_kv < 1 or heads_q % heads_kv:+ raise ValueError("heads_q must be divisible by heads_kv")+ qhead_per_kvhead = heads_q // heads_kv+ if qhead_per_kvhead > PAGE_SIZE:+ raise ValueError(+ "the query-heads-per-KV-head ratio must be at most 128"+ )++ expected_query_sf = (32, 4, 1, 4, scale_atoms, heads_kv, rows)+ expected_page_sf = (page_count, 32, 4, 1, 4, scale_atoms, heads_kv)+ if tuple(query_scales.shape) != expected_query_sf:+ raise ValueError(+ f"query_scales must have shape {expected_query_sf}"+ )+ query_head_bytes = scale_atoms * 512+ expected_query_sf_strides = (+ 16,+ 4,+ heads_q * query_head_bytes,+ 1,+ 512,+ query_head_bytes,+ heads_q * query_head_bytes,+ )+ if query_scales.stride() != expected_query_sf_strides:+ raise ValueError(+ "query_scales must use the layout returned by quantize_query"+ )+ if query_scales_second is not None:+ _require_cuda_tensor(+ query_scales_second, "query_scales_second", device=device+ )+ if tuple(query_scales_second.shape) != expected_query_sf:+ raise ValueError(+ f"query_scales_second must have shape {expected_query_sf}"+ )+ if query_scales_second.stride() != expected_query_sf_strides:+ raise ValueError(+ "query_scales_second must use the layout returned by "+ "quantize_query"+ )+ if tuple(key_scales.shape) != expected_page_sf:+ raise ValueError(f"key_scales must have shape {expected_page_sf}")+ if tuple(value_scales.shape) != expected_page_sf:+ raise ValueError(+ f"value_scales must have shape {expected_page_sf}"+ )+ query_scales = _as_scale_bytes(query_scales, "query_scales", scale_format)+ if query_scales_second is not None:+ query_scales_second = _as_scale_bytes(+ query_scales_second, "query_scales_second", scale_format+ )+ key_scales = _as_scale_dtype(key_scales, "key_scales", scale_format)+ value_scales = _as_scale_dtype(value_scales, "value_scales", scale_format)+ if query_scales.storage_offset() != 0:+ raise ValueError("query_scales must start at storage offset zero")+ if (+ query_scales_second is not None+ and query_scales_second.storage_offset() != 0+ ):+ raise ValueError(+ "query_scales_second must start at storage offset zero"+ )+ key_scales = _page_scales_for_kernel(key_scales, "key_scales")+ value_scales = _page_scales_for_kernel(value_scales, "value_scales")++ if (+ fp4_page_table.dtype is not torch.int32+ or fp4_page_table.ndim != 2+ or fp4_page_table.shape[0] != rows+ or fp4_page_table.stride(-1) != 1+ ):+ raise ValueError(+ "fp4_page_table must be row-major INT32 with shape "+ "[rows, max_pages]"+ )+ if (+ seqused_fp4.dtype is not torch.int32+ or tuple(seqused_fp4.shape) != (rows,)+ or not seqused_fp4.is_contiguous()+ ):+ raise ValueError(+ "seqused_fp4 must be contiguous INT32 with shape [rows]"+ )+ max_fp4_tokens = fp4_page_table.shape[1] * PAGE_SIZE+ # The tagged-metadata producer owns consumed-prefix ID validation. Do not+ # scan unused columns here: persistent vLLM scratch may retain poison or+ # stale IDs beyond the prefix selected by seqused_fp4.+ if not trusted_metadata:+ _check_device_values(+ (seqused_fp4 < 0)+ | (seqused_fp4 > max_fp4_tokens)+ | (seqused_fp4 % PAGE_SIZE != 0),+ "seqused_fp4 values must be page-aligned and in "+ f"[0, {max_fp4_tokens}]",+ )++ residual_values = (+ residual_key_pages_bf16,+ residual_value_pages_bf16,+ residual_page_ids,+ seqused_residual,+ )+ has_residual = any(value is not None for value in residual_values)+ if has_residual and not all(+ value is not None for value in residual_values+ ):+ raise ValueError(+ "residual_key_pages_bf16, residual_value_pages_bf16, "+ "residual_page_ids, and seqused_residual must be provided together"+ )+ if has_residual:+ if query_padded_bf16 is None:+ raise ValueError(+ "pre-quantized query with a BF16 residual requires "+ "query_padded_bf16 support; use the BF16 query path"+ )+ _require_cuda_tensor(+ query_padded_bf16, "query_padded_bf16", device=device+ )+ if (+ query_padded_bf16.dtype is not torch.bfloat16+ or tuple(query_padded_bf16.shape) != expected_padded_shape+ or not query_padded_bf16.is_contiguous()+ ):+ raise ValueError(+ f"query_padded_bf16 must be contiguous BF16 with shape "+ f"{expected_padded_shape}"+ )+ assert residual_key_pages_bf16 is not None+ assert residual_value_pages_bf16 is not None+ assert residual_page_ids is not None+ assert seqused_residual is not None+ for tensor, name in (+ (residual_key_pages_bf16, "residual_key_pages_bf16"),+ (residual_value_pages_bf16, "residual_value_pages_bf16"),+ (residual_page_ids, "residual_page_ids"),+ (seqused_residual, "seqused_residual"),+ ):+ _require_cuda_tensor(tensor, name, device=device)+ if residual_key_pages_bf16.ndim != 4:+ raise ValueError(+ "residual_key_pages_bf16 must be contiguous BF16 with "+ "shape [pages, 128, heads_kv, 128]"+ )+ residual_pages = residual_key_pages_bf16.shape[0]+ residual_rows = PAGE_SIZE + (+ SINK_TOKENS if seqused_sink is not None else 0+ )+ expected_residual_shape = (+ residual_pages,+ residual_rows,+ heads_kv,+ HEAD_DIM,+ )+ if (+ residual_key_pages_bf16.dtype is not torch.bfloat16+ or tuple(residual_key_pages_bf16.shape)+ != expected_residual_shape+ or not residual_key_pages_bf16.is_contiguous()+ ):+ raise ValueError(+ "residual_key_pages_bf16 must be contiguous BF16 with "+ f"shape [pages, {residual_rows}, heads_kv, 128]"+ )+ if (+ residual_value_pages_bf16.dtype is not torch.bfloat16+ or tuple(residual_value_pages_bf16.shape)+ != expected_residual_shape+ or not residual_value_pages_bf16.is_contiguous()+ ):+ raise ValueError(+ "residual_value_pages_bf16 must match the residual K cache"+ )+ if (+ residual_page_ids.dtype is not torch.int32+ or tuple(residual_page_ids.shape) != (rows,)+ or not residual_page_ids.is_contiguous()+ ):+ raise ValueError(+ "residual_page_ids must be contiguous INT32 with shape [rows]"+ )+ if (+ seqused_residual.dtype is not torch.int32+ or tuple(seqused_residual.shape) != (rows,)+ or not seqused_residual.is_contiguous()+ ):+ raise ValueError(+ "seqused_residual must be contiguous INT32 with shape [rows]"+ )+ if seqused_sink is not None:+ _require_cuda_tensor(+ seqused_sink, "seqused_sink", device=device+ )+ if (+ seqused_sink.dtype is not torch.int32+ or tuple(seqused_sink.shape) != (rows,)+ or not seqused_sink.is_contiguous()+ ):+ raise ValueError(+ "seqused_sink must be contiguous INT32 with shape [rows]"+ )+ if not trusted_metadata:+ _check_device_values(+ (residual_page_ids < 0)+ | (residual_page_ids >= residual_pages),+ "residual_page_ids contains an out-of-range physical page ID",+ )+ _check_device_values(+ (seqused_residual < 0)+ | (seqused_residual > PAGE_SIZE),+ "seqused_residual values must be in [0, 128]",+ )+ if seqused_sink is not None:+ _check_device_values(+ (seqused_sink < 0) | (seqused_sink > SINK_TOKENS),+ f"seqused_sink values must be in [0, {SINK_TOKENS}]",+ )+ if has_bf16 is not None:+ _require_cuda_tensor(has_bf16, "has_bf16", device=device)+ if (+ has_bf16.dtype is not torch.bool+ or tuple(has_bf16.shape) != (rows,)+ or not has_bf16.is_contiguous()+ ):+ raise ValueError(+ "has_bf16 must be contiguous BOOL with shape [rows]"+ )+ if not trusted_metadata:+ _check_device_values(+ has_bf16 != (seqused_residual > 0),+ "has_bf16 must agree with seqused_residual > 0",+ )+ elif has_bf16 is not None:+ raise ValueError(+ "has_bf16 requires the BF16 residual cache arguments"+ )+ elif seqused_sink is not None:+ raise ValueError(+ "seqused_sink requires the BF16 residual cache arguments"+ )++ # These run before the validate_only exit because the split dispatcher in+ # _kernel.py borrows this function for its contract checks and then writes+ # into the same caller buffer, where nothing else would catch a bad out.+ if out is None and out_indices is not None:+ raise ValueError("out_indices needs the out it indexes into")+ if out is not None:+ _require_cuda_tensor(out, "out", device=device)+ if (+ out.dtype is not torch.bfloat16+ or out.ndim != 3+ or tuple(out.shape[1:]) != (heads_q, HEAD_DIM)+ or not out.is_contiguous()+ ):+ raise ValueError(+ "out must be contiguous BF16 with shape "+ "[output_rows, heads_q, 128]"+ )+ if out_indices is None:+ # Row i of the batch lands in row i of out, so out cannot be+ # shorter than the batch.+ if out.shape[0] < rows:+ raise ValueError(+ f"out needs at least {rows} rows when no out_indices say "+ f"where they go, found {out.shape[0]}"+ )+ else:+ _require_cuda_tensor(out_indices, "out_indices", device=device)+ if (+ out_indices.dtype is not torch.int32+ or tuple(out_indices.shape) != (rows,)+ or not out_indices.is_contiguous()+ ):+ raise ValueError(+ "out_indices must be contiguous INT32 with shape [rows]"+ )+ if not trusted_metadata:+ _check_device_values(+ (out_indices < 0) | (out_indices >= out.shape[0]),+ "out_indices contains an out-of-range output row",+ )++ if validate_only:+ return None++ packed_query_scales = query_scales+ if out is not None:+ # A scatter needs every row it might land on; without one the kernel+ # sizes its grid from this, so it must be exactly the batch.+ output_4d = (out if out_indices is not None else out[:rows]).unsqueeze(+ 1+ )+ else:+ output_4d = torch.empty(+ rows,+ 1,+ heads_q,+ HEAD_DIM,+ dtype=torch.bfloat16,+ device=device,+ )+ with torch.cuda.device(device):+ compiled = _compile_decode(+ query_fp4=query_fp4,+ query_scales=packed_query_scales,+ query_fp4_second=query_fp4_second,+ query_scales_second=query_scales_second,+ query_padded_bf16=query_padded_bf16,+ key_pages_fp4=key_pages_fp4,+ key_scales=key_scales,+ value_pages_fp4=value_pages_fp4,+ value_scales=value_scales,+ fp4_page_table=fp4_page_table,+ seqused_fp4=seqused_fp4,+ output=output_4d,+ softmax_scale=softmax_scale,+ residual_key_pages_bf16=residual_key_pages_bf16,+ residual_value_pages_bf16=residual_value_pages_bf16,+ residual_page_ids=residual_page_ids,+ seqused_residual=seqused_residual,+ seqused_sink=seqused_sink,+ out_indices=out_indices,+ scale_format=scale_format,+ )+ stream = cuda.CUstream(+ torch.cuda.current_stream(device).cuda_stream+ )+ q_pointer = make_ptr(+ cutlass.Float4E2M1FN,+ query_fp4.data_ptr(),+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ q_second_pointer = make_ptr(+ cutlass.Float4E2M1FN,+ (+ query_fp4_second.data_ptr()+ if query_fp4_second is not None+ else query_fp4.data_ptr()+ ),+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ k_pointer = make_ptr(+ cutlass.Float4E2M1FN,+ key_pages_fp4.data_ptr(),+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ v_pointer = make_ptr(+ cutlass.Float4E2M1FN,+ value_pages_fp4.data_ptr(),+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ call_args = (+ q_pointer,+ k_pointer,+ v_pointer,+ output_4d,+ None,+ softmax_scale,+ stream,+ None,+ None,+ None,+ seqused_fp4,+ fp4_page_table,+ None,+ None,+ None,+ None,+ None,+ packed_query_scales,+ key_scales,+ value_scales,+ (+ rows,+ 1,+ heads_q,+ HEAD_DIM,+ ),+ (+ page_count,+ PAGE_SIZE,+ heads_kv,+ HEAD_DIM,+ ),+ (+ page_count,+ PAGE_SIZE,+ heads_kv,+ HEAD_DIM,+ ),+ )+ # Packed pitches scale by two because one byte holds two FP4. The scale+ # slabs are byte-per-element, and _page_scales_for_kernel moved their+ # page axis last.+ call_kwargs: dict[str, Any] = {+ "k_page_stride": _page_stride_bytes(+ key_pages_fp4, "key_pages_fp4"+ ) * 2,+ "v_page_stride": _page_stride_bytes(+ value_pages_fp4, "value_pages_fp4"+ ) * 2,+ "k_sf_page_stride": key_scales.stride()[-1],+ "v_sf_page_stride": value_scales.stride()[-1],+ "mOutIndices": out_indices,+ "mQSecond": q_second_pointer,+ "mSFQSecond": (+ query_scales_second+ if query_scales_second is not None+ else packed_query_scales+ ),+ }+ if has_residual:+ call_kwargs.update(+ mResidualQ=query_padded_bf16,+ mResidualK=None,+ mResidualV=None,+ mResidualSeqUsedK=seqused_residual,+ mResidualKCache=residual_key_pages_bf16,+ mResidualVCache=residual_value_pages_bf16,+ mResidualBlockIds=residual_page_ids,+ mSinkSeqUsedK=seqused_sink,+ )+ compiled(*call_args, **call_kwargs)+ if out is not None:+ # Only a scatter can land rows past the batch prefix, so only then is+ # the whole buffer the answer. Otherwise out[rows:] belongs to other+ # requests, and the split path returns the same prefix.+ return out if out_indices is not None else out[:rows]+ return output_4d[:, 0]+++def decode_fp4_split(+ *,+ query_fp4: torch.Tensor,+ query_scales: torch.Tensor,+ key_pages_fp4: torch.Tensor,+ key_scales: torch.Tensor,+ value_pages_fp4: torch.Tensor,+ value_scales: torch.Tensor,+ fp4_page_table: torch.Tensor,+ seqused_fp4: torch.Tensor,+ softmax_scale: float,+ num_splits: int,+ query_fp4_second: torch.Tensor | None = None,+ query_scales_second: torch.Tensor | None = None,+ query_padded_bf16: torch.Tensor | None = None,+ residual_key_pages_bf16: torch.Tensor | None = None,+ residual_value_pages_bf16: torch.Tensor | None = None,+ residual_page_ids: torch.Tensor | None = None,+ seqused_residual: torch.Tensor | None = None,+ seqused_sink: torch.Tensor | None = None,+ has_bf16: torch.Tensor | None = None,+ out: torch.Tensor | None = None,+ split_partial_scratch: torch.Tensor | None = None,+ split_lse_scratch: torch.Tensor | None = None,+ scale_format: str = "nvfp4",+) -> torch.Tensor:+ """Run split-K decode, with an optional residual tail, and combine partials."""+ if num_splits <= 1:+ return decode_fp4(+ query_fp4=query_fp4,+ query_scales=query_scales,+ query_fp4_second=query_fp4_second,+ query_scales_second=query_scales_second,+ query_padded_bf16=query_padded_bf16,+ key_pages_fp4=key_pages_fp4,+ key_scales=key_scales,+ value_pages_fp4=value_pages_fp4,+ value_scales=value_scales,+ fp4_page_table=fp4_page_table,+ seqused_fp4=seqused_fp4,+ residual_key_pages_bf16=residual_key_pages_bf16,+ residual_value_pages_bf16=residual_value_pages_bf16,+ residual_page_ids=residual_page_ids,+ seqused_residual=seqused_residual,+ seqused_sink=seqused_sink,+ has_bf16=has_bf16,+ softmax_scale=softmax_scale,+ out=out,+ scale_format=scale_format,+ )++ # Reuse the core validator before allocating the split workspace.+ rows, _, heads_q, _ = query_fp4.shape+ device = query_fp4.device+ heads_kv = key_pages_fp4.shape[2]+ if heads_q % heads_kv:+ raise ValueError("heads_q must be divisible by heads_kv")+ if (query_fp4_second is None) != (query_scales_second is None):+ raise ValueError(+ "query_fp4_second and query_scales_second must be provided together"+ )+ for tensor, name in (+ (query_fp4, "query_fp4"),+ (query_scales, "query_scales"),+ (key_pages_fp4, "key_pages_fp4"),+ (key_scales, "key_scales"),+ (value_pages_fp4, "value_pages_fp4"),+ (value_scales, "value_scales"),+ (fp4_page_table, "fp4_page_table"),+ (seqused_fp4, "seqused_fp4"),+ ):+ _require_cuda_tensor(tensor, name, device=device)+ partials = split_partial_count(+ rows,+ heads_kv,+ num_splits,+ residual=residual_key_pages_bf16 is not None,+ sms=_multiprocessor_count(device.index),+ )+ # Neither needs clearing, which is what lets a captured step reuse them+ # without a fill per layer. The kernel writes every LSE slot, -inf for a+ # split that was handed no work, and the combine accumulates a partial+ # only under a positive weight, so an output slot a split never wrote is+ # read at most and never summed, whatever it still holds.+ partial_shape = (partials, rows, 1, heads_q, HEAD_DIM)+ lse_shape = (partials, rows, heads_q, 1)+ output_partial = _split_workspace(+ split_partial_scratch, "split_partial_scratch", partial_shape, device+ )+ lse_partial = _split_workspace(+ split_lse_scratch, "split_lse_scratch", lse_shape, device+ )+ # Both accepted query-scale dtypes must collapse to one tensor signature+ # before compiling, because the compile cache is not keyed on dtype and+ # would otherwise hand back a kernel built for the other one.+ query_scales = _as_scale_bytes(query_scales, "query_scales", scale_format)+ if query_scales_second is not None:+ query_scales_second = _as_scale_bytes(+ query_scales_second, "query_scales_second", scale_format+ )+ key_scales_kernel = _page_scales_for_kernel(+ _as_scale_dtype(key_scales, "key_scales", scale_format), "key_scales"+ )+ value_scales_kernel = _page_scales_for_kernel(+ _as_scale_dtype(value_scales, "value_scales", scale_format), "value_scales"+ )+ with torch.cuda.device(device):+ compiled = _compile_split_decode(+ query_fp4=query_fp4,+ query_scales=query_scales,+ query_scales_second=query_scales_second,+ key_pages_fp4=key_pages_fp4,+ key_scales=key_scales_kernel,+ value_pages_fp4=value_pages_fp4,+ value_scales=value_scales_kernel,+ fp4_page_table=fp4_page_table,+ seqused_fp4=seqused_fp4,+ output_partial=output_partial,+ lse_partial=lse_partial,+ softmax_scale=softmax_scale,+ num_splits=num_splits,+ bf16_split=partials > num_splits,+ query_padded_bf16=query_padded_bf16,+ residual_key_pages_bf16=residual_key_pages_bf16,+ residual_value_pages_bf16=residual_value_pages_bf16,+ residual_page_ids=residual_page_ids,+ seqused_residual=seqused_residual,+ seqused_sink=seqused_sink,+ scale_format=scale_format,+ )+ stream = cuda.CUstream(torch.cuda.current_stream(device).cuda_stream)+ fp4_ptr = lambda tensor: make_ptr(+ cutlass.Float4E2M1FN,+ tensor.data_ptr(),+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ residual_kwargs: dict[str, Any] = {}+ if residual_key_pages_bf16 is not None:+ residual_kwargs = dict(+ mResidualQ=query_padded_bf16,+ mResidualK=None,+ mResidualV=None,+ mResidualSeqUsedK=seqused_residual,+ mResidualKCache=residual_key_pages_bf16,+ mResidualVCache=residual_value_pages_bf16,+ mResidualBlockIds=residual_page_ids,+ mSinkSeqUsedK=seqused_sink,+ )+ compiled(+ fp4_ptr(query_fp4),+ fp4_ptr(key_pages_fp4),+ fp4_ptr(value_pages_fp4),+ output_partial,+ lse_partial,+ softmax_scale,+ stream,+ None,+ None,+ None,+ seqused_fp4,+ fp4_page_table,+ None,+ None,+ None,+ None,+ None,+ query_scales,+ key_scales_kernel,+ value_scales_kernel,+ (+ rows,+ 1,+ heads_q,+ HEAD_DIM,+ ),+ (+ key_pages_fp4.shape[0],+ PAGE_SIZE,+ heads_kv,+ HEAD_DIM,+ ),+ (+ key_pages_fp4.shape[0],+ PAGE_SIZE,+ heads_kv,+ HEAD_DIM,+ ),+ k_page_stride=_page_stride_bytes(key_pages_fp4, "key_pages_fp4") * 2,+ v_page_stride=_page_stride_bytes(value_pages_fp4, "value_pages_fp4") * 2,+ k_sf_page_stride=key_scales_kernel.stride()[-1],+ v_sf_page_stride=value_scales_kernel.stride()[-1],+ mOutIndices=None,+ mQSecond=fp4_ptr(+ query_fp4_second if query_fp4_second is not None else query_fp4+ ),+ mSFQSecond=(+ query_scales_second+ if query_scales_second is not None+ else query_scales+ ),+ **residual_kwargs,+ )+ from .split_k_combine import combine_split_partials++ output = combine_split_partials(+ output_partial,+ lse_partial.transpose(-1, -2),+ # The combine carries the single-token axis the partials have, so a+ # caller's [rows, heads, dim] buffer is viewed with it put back.+ out=None if out is None else out[:rows].unsqueeze(1),+ )+ return output[:, 0]diff --git a/src/nvfp4_decode_kernel/_fa4/__init__.py b/src/nvfp4_decode_kernel/_fa4/__init__.pynew file mode 100644index 0000000..b838835--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/__init__.py@@ -0,0 +1 @@+"""Private CuTeDSL helpers for the SM100 FP4 decode kernel."""diff --git a/src/nvfp4_decode_kernel/_fa4/blackwell_helpers.py b/src/nvfp4_decode_kernel/_fa4/blackwell_helpers.pynew file mode 100644index 0000000..262fdc3--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/blackwell_helpers.py@@ -0,0 +1,1082 @@+# Copyright (c) 2025, Tri Dao.+from typing import Optional, Tuple++import cutlass+import cutlass.cute as cute+from cutlass import Int32, Boolean, const_expr, Float32+from cutlass.cute.nvgpu import tcgen05+from cutlass._mlir.dialects import llvm+from cutlass.cutlass_dsl import T++from . import mma_sm100_desc as sm100_desc+from .utils import parse_swizzle_from_pointer+++def gemm_w_idx(+ tiled_mma: cute.TiledMma,+ acc: cute.Tensor,+ tCrA: cute.Tensor,+ tCrB: cute.Tensor,+ A_idx: Optional[Int32] = None,+ B_idx: Optional[Int32] = None,+ zero_init: bool | Boolean = False,+ swap_AB: bool = False,+) -> None:+ if const_expr(swap_AB):+ return gemm_w_idx(+ tiled_mma, acc, tCrB, tCrA, B_idx, A_idx, zero_init=zero_init, swap_AB=False+ )+ else:+ rA = tCrA if const_expr(A_idx is None) else tCrA[None, None, None, A_idx]+ rB = tCrB if const_expr(B_idx is None) else tCrB[None, None, None, B_idx]+ mma_atom = cute.make_mma_atom(tiled_mma.op)+ for k in cutlass.range_constexpr(cute.size(tCrA.shape[2])):+ mma_atom.set(tcgen05.Field.ACCUMULATE, not zero_init or k != 0)+ cute.gemm(mma_atom, acc, rA[None, None, k], rB[None, None, k], acc)+++def gemm_ptx_w_idx(+ tiled_mma: cute.TiledMma,+ acc: cute.Tensor,+ tCrA: cute.Tensor,+ tCrB: cute.Tensor,+ sA: Optional[cute.Tensor],+ sB: cute.Tensor,+ A_idx: Optional[Int32] = None,+ B_idx: Optional[Int32] = None,+ zero_init: bool | Boolean = False,+ **kwargs,+) -> None:+ rA = tCrA if const_expr(A_idx is None) else tCrA[None, None, None, A_idx]+ rB = tCrB if const_expr(B_idx is None) else tCrB[None, None, None, B_idx]+ sA_cur = None+ if const_expr(sA is not None):+ sA_cur = sA if const_expr(A_idx is None) else sA[None, None, None, A_idx]+ sB_cur = sB if const_expr(B_idx is None) else sB[None, None, None, B_idx]+ mma_atom = cute.make_mma_atom(tiled_mma.op)+ acc_tmem_addr = acc.iterator.toint()+ gemm_ptx_partial(+ mma_atom.op, acc_tmem_addr, rA, rB, sA_cur, sB_cur, zero_init=zero_init, **kwargs+ )+++def gemm(+ tiled_mma: cute.TiledMma,+ acc: cute.Tensor,+ tCrA: cute.Tensor,+ tCrB: cute.Tensor,+ zero_init: bool | Boolean = False,+) -> cute.TiledMma:+ for k in cutlass.range_constexpr(cute.size(tCrA.shape[2])):+ tiled_mma.set(tcgen05.Field.ACCUMULATE, not zero_init or k != 0)+ cute.gemm(tiled_mma, acc, tCrA[None, None, k], tCrB[None, None, k], acc)+ return tiled_mma+++def i64_to_i32x2(i: int) -> Tuple[int, int]:+ """Convert a 64-bit integer to a tuple of two 32-bit integers."""+ return i & 0xFFFF_FFFF, (i >> 32) & 0xFFFF_FFFF+++def gemm_ptx(+ op: cute.nvgpu.tcgen05.mma.MmaOp,+ acc: cute.Tensor,+ tCrA: cute.Tensor,+ tCrB: cute.Tensor,+ sA: Optional[cute.Tensor],+ sB: cute.Tensor,+ zero_init: bool | Boolean = False,+) -> None:+ is_ts = op.a_src == cute.nvgpu.tcgen05.OperandSource.TMEM+ if const_expr(not is_ts):+ assert sA is not None, "sA must be provided when a_src is not TMEM"+ sA_layout = sA.layout if sA is not None else None+ sB_layout = sB.layout+ idesc: int = const_expr(sm100_desc.mma_op_to_idesc(op))+ if const_expr(not is_ts):+ sA_swizzle = parse_swizzle_from_pointer(sA.iterator)+ smem_desc_base_a: int = const_expr(+ sm100_desc.make_smem_desc_base(+ cute.recast_layout(128, op.a_dtype.width, sA_layout[0]),+ sA_swizzle,+ sm100_desc.Major.K+ if const_expr(op.a_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K)+ else sm100_desc.Major.MN,+ )+ )+ smem_desc_base_a_lo, smem_desc_a_hi = i64_to_i32x2(smem_desc_base_a)+ smem_desc_base_a_lo = const_expr(smem_desc_base_a_lo)+ smem_desc_a_hi = const_expr(smem_desc_a_hi)+ else:+ smem_desc_base_a = None+ smem_desc_base_a_lo, smem_desc_a_hi = None, None+ sB_swizzle = parse_swizzle_from_pointer(sB.iterator)+ smem_desc_base_b: int = const_expr(+ sm100_desc.make_smem_desc_base(+ cute.recast_layout(128, op.b_dtype.width, sB_layout[0]),+ sB_swizzle,+ sm100_desc.Major.K+ if const_expr(op.b_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K)+ else sm100_desc.Major.MN,+ )+ )+ smem_desc_base_b_lo, smem_desc_b_hi = i64_to_i32x2(smem_desc_base_b)+ smem_desc_base_b_lo = const_expr(smem_desc_base_b_lo)+ smem_desc_b_hi = const_expr(smem_desc_b_hi)++ if const_expr(not is_ts):+ smem_desc_start_a_lo = Int32(smem_desc_base_a_lo) | sm100_desc.make_smem_desc_start_addr(+ sA[None, None, 0].iterator+ )+ else:+ smem_desc_start_a_lo = None+ smem_desc_start_b_lo = Int32(smem_desc_base_b_lo) | sm100_desc.make_smem_desc_start_addr(+ sB[None, None, 0].iterator+ )+ for k in cutlass.range_constexpr(cute.size(tCrA.shape[2])):+ if const_expr(not is_ts):+ smem_desc_a_lo = smem_desc_start_a_lo + (+ (cute.crd2idx((0, 0, k), sA_layout) * sA.element_type.width // 8) >> 4+ )+ smem_desc_b_lo = smem_desc_start_b_lo + (+ (cute.crd2idx((0, 0, k), sB_layout) * sB.element_type.width // 8) >> 4+ )+ with cute.arch.elect_one():+ if const_expr(not is_ts):+ llvm.inline_asm(+ None,+ [+ acc.iterator.toint().ir_value(),+ smem_desc_a_lo.ir_value(),+ smem_desc_b_lo.ir_value(),+ Int32(not zero_init or k != 0).ir_value(),+ ],+ "{\n\t"+ ".reg .pred p;\n\t"+ ".reg .b64 smem_desc_a, smem_desc_b;\n\t"+ ".reg .b32 idesc;\n\t"+ f"mov.b32 idesc, {hex(idesc)};\n\t"+ f"mov.b64 smem_desc_a, {{$1, {hex(smem_desc_a_hi)}}};\n\t"+ f"mov.b64 smem_desc_b, {{$2, {hex(smem_desc_b_hi)}}};\n\t"+ "setp.ne.b32 p, $3, 0;\n\t"+ f"tcgen05.mma.cta_group::1.kind::f16 [$0], smem_desc_a, smem_desc_b, idesc, p;\n\t"+ "}\n",+ "r,r,r,r",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ else:+ llvm.inline_asm(+ None,+ [+ acc.iterator.toint().ir_value(),+ tCrA[None, None, k].iterator.toint().ir_value(),+ smem_desc_b_lo.ir_value(),+ Int32(not zero_init or k != 0).ir_value(),+ ],+ "{\n\t"+ ".reg .pred p;\n\t"+ ".reg .b64 smem_desc_b;\n\t"+ f"mov.b64 smem_desc_b, {{$2, {hex(smem_desc_b_hi)}}};\n\t"+ "setp.ne.b32 p, $3, 0;\n\t"+ f"tcgen05.mma.cta_group::1.kind::f16 [$0], [$1], smem_desc_b, {hex(idesc)}, p;\n\t"+ "}\n",+ "r,r,r,r",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+++def gemm_ptx_loop(+ op: cute.nvgpu.tcgen05.mma.MmaOp,+ acc: cute.Tensor,+ tCrA: cute.Tensor,+ tCrB: cute.Tensor,+ sA: Optional[cute.Tensor],+ sB: cute.Tensor,+ zero_init: bool | Boolean = False,+) -> None:+ is_ts = op.a_src == cute.nvgpu.tcgen05.OperandSource.TMEM+ if const_expr(not is_ts):+ assert sA is not None, "sA must be provided when a_src is not TMEM"+ sA_layout = sA.layout if sA is not None else tCrA.layout+ sB_layout = sB.layout+ idesc: int = const_expr(sm100_desc.mma_op_to_idesc(op))+ if const_expr(not is_ts):+ sA_swizzle = parse_swizzle_from_pointer(sA.iterator)+ smem_desc_base_a: int = const_expr(+ sm100_desc.make_smem_desc_base(+ cute.recast_layout(128, op.a_dtype.width, sA_layout[0]),+ sA_swizzle,+ sm100_desc.Major.K+ if const_expr(op.a_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K)+ else sm100_desc.Major.MN,+ )+ )+ smem_desc_base_a_lo, smem_desc_a_hi = i64_to_i32x2(smem_desc_base_a)+ smem_desc_base_a_lo = const_expr(smem_desc_base_a_lo)+ smem_desc_a_hi = const_expr(smem_desc_a_hi)+ else:+ smem_desc_base_a = None+ smem_desc_base_a_lo, smem_desc_a_hi = None, None+ sB_swizzle = parse_swizzle_from_pointer(sB.iterator)+ smem_desc_base_b: int = const_expr(+ sm100_desc.make_smem_desc_base(+ cute.recast_layout(128, op.b_dtype.width, sB_layout[0]),+ sB_swizzle,+ sm100_desc.Major.K+ if const_expr(op.b_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K)+ else sm100_desc.Major.MN,+ )+ )+ smem_desc_base_b_lo, smem_desc_b_hi = i64_to_i32x2(smem_desc_base_b)+ smem_desc_base_b_lo = const_expr(smem_desc_base_b_lo)+ smem_desc_b_hi = const_expr(smem_desc_b_hi)++ if const_expr(not is_ts):+ offset_a = [+ (cute.crd2idx((0, 0, k), sA_layout) * sA.element_type.width // 8) >> 4+ for k in cutlass.range_constexpr(cute.size(tCrA.shape[2]))+ ]+ else:+ offset_a = [+ cute.crd2idx((0, 0, k), sA_layout) * op.a_dtype.width // 32+ for k in cutlass.range_constexpr(cute.size(tCrA.shape[2]))+ ]+ offset_a_diff = [+ offset_a[k] - offset_a[k - 1] for k in cutlass.range_constexpr(1, cute.size(tCrA.shape[2]))+ ]+ offset_b = [+ (cute.crd2idx((0, 0, k), sB_layout) * sB.element_type.width // 8) >> 4+ for k in cutlass.range_constexpr(cute.size(tCrB.shape[2]))+ ]+ offset_b_diff = [+ offset_b[k] - offset_b[k - 1] for k in cutlass.range_constexpr(1, cute.size(tCrB.shape[2]))+ ]++ if const_expr(not is_ts):+ smem_desc_start_a_lo = Int32(+ smem_desc_base_a_lo | sm100_desc.make_smem_desc_start_addr(sA[None, None, 0].iterator)+ )+ else:+ smem_desc_start_a_lo = None+ smem_desc_start_b_lo = Int32(+ smem_desc_base_b_lo | sm100_desc.make_smem_desc_start_addr(sB[None, None, 0].iterator)+ )+ pred_str = "p" if isinstance(zero_init, Boolean) else "0" if zero_init else "1"+ if const_expr(not is_ts):+ llvm.inline_asm(+ None,+ [+ acc.iterator.toint().ir_value(),+ Int32(cute.arch.make_warp_uniform(smem_desc_start_a_lo)).ir_value(),+ Int32(cute.arch.make_warp_uniform(smem_desc_start_b_lo)).ir_value(),+ Int32(not zero_init).ir_value(),+ ],+ "{\n\t"+ ".reg .pred leader_thread;\n\t"+ ".reg .pred p;\n\t"+ ".reg .b32 idesc;\n\t"+ ".reg .b32 smem_desc_a_lo, smem_desc_b_lo;\n\t"+ ".reg .b32 smem_desc_a_hi, smem_desc_b_hi;\n\t"+ ".reg .b64 smem_desc_a, smem_desc_b;\n\t"+ "elect.sync _|leader_thread, -1;\n\t"+ f"mov.b32 idesc, {hex(idesc)};\n\t"+ "mov.b32 smem_desc_a_lo, $1;\n\t"+ "mov.b32 smem_desc_b_lo, $2;\n\t"+ f"mov.b32 smem_desc_a_hi, {hex(smem_desc_a_hi)};\n\t"+ f"mov.b32 smem_desc_b_hi, {hex(smem_desc_b_hi)};\n\t"+ f"mov.b64 smem_desc_a, {{smem_desc_a_lo, smem_desc_a_hi}};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t"+ "setp.ne.b32 p, $3, 0;\n\t"+ f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [$0], smem_desc_a, smem_desc_b, idesc, {pred_str};\n\t"+ + "".join(+ (+ f"add.u32 smem_desc_a_lo, smem_desc_a_lo, {hex(offset_a_diff[k - 1])};\n\t"+ f"add.u32 smem_desc_b_lo, smem_desc_b_lo, {hex(offset_b_diff[k - 1])};\n\t"+ f"mov.b64 smem_desc_a, {{smem_desc_a_lo, smem_desc_a_hi}};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t"+ f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [$0], smem_desc_a, smem_desc_b, idesc, 1;\n\t"+ )+ for k in cutlass.range_constexpr(1, cute.size(tCrA.shape[2]))+ )+ + "}\n",+ "r,r,r,r",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ else:+ llvm.inline_asm(+ None,+ [+ acc.iterator.toint().ir_value(),+ Int32(tCrA[None, None, 0].iterator.toint()).ir_value(),+ Int32(smem_desc_start_b_lo).ir_value(),+ Int32(not zero_init).ir_value(),+ ],+ "{\n\t"+ ".reg .pred leader_thread;\n\t"+ ".reg .pred p;\n\t"+ ".reg .b32 idesc;\n\t"+ ".reg .b32 tmem_a;\n\t"+ ".reg .b32 smem_desc_b_lo;\n\t"+ ".reg .b32 smem_desc_b_hi;\n\t"+ ".reg .b64 smem_desc_b;\n\t"+ "elect.sync _|leader_thread, -1;\n\t"+ f"mov.b32 idesc, {hex(idesc)};\n\t"+ "mov.b32 tmem_a, $1;\n\t"+ "mov.b32 smem_desc_b_lo, $2;\n\t"+ f"mov.b32 smem_desc_b_hi, {hex(smem_desc_b_hi)};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t"+ "setp.ne.b32 p, $3, 0;\n\t"+ f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [$0], [tmem_a], smem_desc_b, idesc, {pred_str};\n\t"+ + "".join(+ (+ # f"add.u32 tmem_a, tmem_a, {hex(offset_a_diff[k - 1])};\n\t"+ f"add.u32 smem_desc_b_lo, smem_desc_b_lo, {hex(offset_b_diff[k - 1])};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t"+ # f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [$0], [tmem_a], smem_desc_b, idesc, 1;\n\t"+ f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [$0], [tmem_a + {hex(offset_a[k])}], smem_desc_b, idesc, 1;\n\t"+ )+ for k in cutlass.range_constexpr(1, cute.size(tCrA.shape[2]))+ )+ + "}\n",+ "r,r,r,r",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+++def gemm_ptx_partial(+ op: cute.nvgpu.tcgen05.mma.MmaOp,+ acc_tmem_addr: Int32,+ tCrA: cute.Tensor,+ tCrB: cute.Tensor,+ sA: Optional[cute.Tensor],+ sB: cute.Tensor,+ mbar_ptr: Optional[cutlass.Pointer] = None,+ mbar_phase: Optional[Int32] = None,+ zero_init: bool | Boolean = False,+ # sA_offset: Int32 = 0,+ # acc_offset: Int32 = 0,+ tA_addr: Optional[Int32] = None,+ pre_mbar_tiles: Optional[cutlass.Constexpr[int]] = None,+) -> None:+ # acc_tmem_addr += acc_offset+ is_ts = op.a_src == cute.nvgpu.tcgen05.OperandSource.TMEM+ if const_expr(not is_ts):+ assert sA is not None, "sA must be provided when a_src is not TMEM"+ sA_layout = sA.layout if sA is not None else tCrA.layout+ sB_layout = sB.layout+ idesc: int = const_expr(sm100_desc.mma_op_to_idesc(op))+ if const_expr(not is_ts):+ sA_swizzle = parse_swizzle_from_pointer(sA.iterator)+ smem_desc_base_a: int = const_expr(+ sm100_desc.make_smem_desc_base(+ cute.recast_layout(128, op.a_dtype.width, sA_layout[0]),+ sA_swizzle,+ sm100_desc.Major.K+ if const_expr(op.a_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K)+ else sm100_desc.Major.MN,+ )+ )+ smem_desc_base_a_lo, smem_desc_a_hi = i64_to_i32x2(smem_desc_base_a)+ smem_desc_base_a_lo = const_expr(smem_desc_base_a_lo)+ smem_desc_a_hi = const_expr(smem_desc_a_hi)+ else:+ smem_desc_base_a = None+ smem_desc_base_a_lo, smem_desc_a_hi = None, None+ sB_swizzle = parse_swizzle_from_pointer(sB.iterator)+ smem_desc_base_b: int = const_expr(+ sm100_desc.make_smem_desc_base(+ cute.recast_layout(128, op.b_dtype.width, sB_layout[0]),+ sB_swizzle,+ sm100_desc.Major.K+ if const_expr(op.b_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K)+ else sm100_desc.Major.MN,+ )+ )+ smem_desc_base_b_lo, smem_desc_b_hi = i64_to_i32x2(smem_desc_base_b)+ smem_desc_base_b_lo = const_expr(smem_desc_base_b_lo)+ smem_desc_b_hi = const_expr(smem_desc_b_hi)++ tCrA_layout = (+ tCrA.layout+ if const_expr(not is_ts)+ else cute.recast_layout(32, tCrA.element_type.width, tCrA.layout)+ )+ offset_a = [cute.crd2idx((0, 0, k), tCrA_layout) for k in range(cute.size(tCrA.shape[2]))]+ offset_a_diff = [offset_a[k] - offset_a[k - 1] for k in range(1, cute.size(tCrA.shape[2]))]+ offset_b = [cute.crd2idx((0, 0, k), tCrB.layout) for k in range(cute.size(tCrB.shape[2]))]+ offset_b_diff = [offset_b[k] - offset_b[k - 1] for k in range(1, cute.size(tCrB.shape[2]))]++ if const_expr(not is_ts):+ smem_desc_start_a_lo = Int32(+ smem_desc_base_a_lo | sm100_desc.make_smem_desc_start_addr(sA[None, None, 0].iterator)+ )+ # ) + sA_offset+ else:+ smem_desc_start_a_lo = None+ smem_desc_start_b_lo = Int32(+ smem_desc_base_b_lo | sm100_desc.make_smem_desc_start_addr(sB[None, None, 0].iterator)+ )+ pred_str = "p" if isinstance(zero_init, Boolean) else "0" if zero_init else "1"+ if const_expr(not is_ts):+ assert mbar_ptr is None, "mbar_ptr must be None when a_src is not TMEM"+ llvm.inline_asm(+ None,+ [+ # acc.iterator.toint().ir_value(),+ Int32(cute.arch.make_warp_uniform(smem_desc_start_a_lo)).ir_value(),+ Int32(cute.arch.make_warp_uniform(smem_desc_start_b_lo)).ir_value(),+ Int32(not zero_init).ir_value(),+ Int32(cute.arch.make_warp_uniform(acc_tmem_addr)).ir_value(),+ ],+ "{\n\t"+ ".reg .pred leader_thread;\n\t"+ ".reg .pred p;\n\t"+ ".reg .b32 idesc;\n\t"+ ".reg .b32 tmem_acc;\n\t"+ ".reg .b32 smem_desc_a_lo_start, smem_desc_b_lo_start;\n\t"+ ".reg .b32 smem_desc_a_lo, smem_desc_b_lo;\n\t"+ ".reg .b32 smem_desc_a_hi, smem_desc_b_hi;\n\t"+ ".reg .b64 smem_desc_a, smem_desc_b;\n\t"+ "elect.sync _|leader_thread, -1;\n\t"+ f"mov.b32 idesc, {hex(idesc)};\n\t"+ # f"mov.b32 tmem_acc, {hex(acc_tmem_addr)};\n\t"+ f"mov.b32 tmem_acc, $3;\n\t"+ "mov.b32 smem_desc_a_lo_start, $0;\n\t"+ "mov.b32 smem_desc_b_lo_start, $1;\n\t"+ f"mov.b32 smem_desc_a_hi, {hex(smem_desc_a_hi)};\n\t"+ f"mov.b32 smem_desc_b_hi, {hex(smem_desc_b_hi)};\n\t"+ f"mov.b64 smem_desc_a, {{smem_desc_a_lo_start, smem_desc_a_hi}};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo_start, smem_desc_b_hi}};\n\t"+ "setp.ne.b32 p, $2, 0;\n\t"+ f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], smem_desc_a, smem_desc_b, idesc, {pred_str};\n\t"+ + "".join(+ (+ # f"add.u32 smem_desc_a_lo, smem_desc_a_lo, {hex(offset_a_diff[k - 1])};\n\t"+ # f"add.u32 smem_desc_b_lo, smem_desc_b_lo, {hex(offset_b_diff[k - 1])};\n\t"+ f"add.u32 smem_desc_a_lo, smem_desc_a_lo_start, {hex(offset_a[k])};\n\t"+ f"add.u32 smem_desc_b_lo, smem_desc_b_lo_start, {hex(offset_b[k])};\n\t"+ f"mov.b64 smem_desc_a, {{smem_desc_a_lo, smem_desc_a_hi}};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t"+ f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], smem_desc_a, smem_desc_b, idesc, 1;\n\t"+ )+ for k in range(1, cute.size(tCrA.shape[2]))+ )+ + "}\n",+ # "r,r,r",+ "r,r,r,r",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ else:+ # For TS gemm, somehow tCrA.iterator.toint() returns 0 no matter what, so we need to+ # explicitly pass in the tA_addr for correctness.+ tA_addr = tCrA[None, None, 0].iterator.toint() if tA_addr is None else tA_addr+ input_args = [+ # Int32(cute.arch.make_warp_uniform(tCrA[None, None, 0].iterator.toint())).ir_value(),+ Int32(cute.arch.make_warp_uniform(tA_addr)).ir_value(),+ Int32(cute.arch.make_warp_uniform(smem_desc_start_b_lo)).ir_value(),+ Int32(not zero_init).ir_value(),+ Int32(cute.arch.make_warp_uniform(acc_tmem_addr)).ir_value(),+ ]+ if const_expr(mbar_ptr is not None):+ assert mbar_phase is not None, "mbar_phase must be provided when mbar_ptr is not None"+ input_args.append(mbar_ptr.toint().ir_value())+ input_args.append(Int32(mbar_phase).ir_value())+ mbar_wait_str = (+ ".reg .pred P1; \n\t"+ "LAB_WAIT: \n\t"+ "mbarrier.try_wait.parity.shared::cta.b64 P1, [$4], $5, 10000000; \n\t"+ "@P1 bra DONE; \n\t"+ "bra LAB_WAIT; \n\t"+ "DONE: \n\t"+ )+ else:+ mbar_wait_str = ""+ llvm.inline_asm(+ None,+ input_args,+ "{\n\t"+ ".reg .pred leader_thread;\n\t"+ ".reg .pred p;\n\t"+ ".reg .b32 idesc;\n\t"+ ".reg .b32 tmem_acc;\n\t"+ ".reg .b32 tmem_a;\n\t"+ ".reg .b32 smem_desc_b_lo_start;\n\t"+ ".reg .b32 smem_desc_b_lo;\n\t"+ ".reg .b32 smem_desc_b_hi;\n\t"+ ".reg .b64 smem_desc_b;\n\t"+ "elect.sync _|leader_thread, -1;\n\t"+ f"mov.b32 idesc, {hex(idesc)};\n\t"+ # f"mov.b32 tmem_acc, {hex(acc_tmem_addr)};\n\t"+ f"mov.b32 tmem_acc, $3;\n\t"+ f"mov.b32 tmem_a, $0;\n\t"+ f"mov.b32 smem_desc_b_lo_start, $1;\n\t"+ f"mov.b32 smem_desc_b_hi, {hex(smem_desc_b_hi)};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo_start, smem_desc_b_hi}};\n\t"+ "setp.ne.b32 p, $2, 0;\n\t"+ f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], [tmem_a], smem_desc_b, idesc, {pred_str};\n\t"+ + "".join(+ (+ # f"add.u32 tmem_a, tmem_a, {hex(offset_a_diff[k - 1])};\n\t"+ # f"add.u32 smem_desc_b_lo, smem_desc_b_lo, {hex(offset_b_diff[k - 1])};\n\t"+ f"add.u32 smem_desc_b_lo, smem_desc_b_lo_start, {hex(offset_b[k])};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t"+ # f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], [tmem_a], smem_desc_b, idesc, 1;\n\t"+ f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], [tmem_a + {hex(offset_a[k])}], smem_desc_b, idesc, 1;\n\t"+ )+ for k in range(+ 1,+ cute.size(tCrA.shape[2])+ if const_expr(mbar_ptr is None)+ else (pre_mbar_tiles if const_expr(pre_mbar_tiles is not None) else cute.size(tCrA.shape[2]) // 4 * 3),+ )+ )+ + mbar_wait_str+ + (+ "".join(+ (+ f"add.u32 smem_desc_b_lo, smem_desc_b_lo_start, {hex(offset_b[k])};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t"+ f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], [tmem_a + {hex(offset_a[k])}], smem_desc_b, idesc, 1;\n\t"+ )+ for k in range(max(1, pre_mbar_tiles if const_expr(pre_mbar_tiles is not None) else cute.size(tCrA.shape[2]) // 4 * 3), cute.size(tCrA.shape[2]))+ )+ if const_expr(mbar_ptr is not None)+ else ""+ )+ + "}\n",+ "r,r,r,r" if const_expr(mbar_ptr is None) else "r,r,r,r,r,r",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+++def gemm_ptx_partial_fp4(+ op: cute.nvgpu.tcgen05.mma.MmaOp,+ acc_tmem_addr: Int32,+ tCrA: cute.Tensor,+ tCrB: cute.Tensor,+ sA: Optional[cute.Tensor],+ sB: cute.Tensor,+ tScaleA: cute.Tensor,+ tScaleB: cute.Tensor,+ mbar_ptr: Optional[cutlass.Pointer] = None,+ mbar_phase: Optional[Int32] = None,+ zero_init: bool | Boolean = False,+ # sA_offset: Int32 = 0,+ # acc_offset: Int32 = 0,+ tA_addr: Optional[Int32] = None,+ pre_mbar_tiles: Optional[cutlass.Constexpr[int]] = None,+) -> None:+ # acc_tmem_addr += acc_offset+ is_ts = op.a_src == cute.nvgpu.tcgen05.OperandSource.TMEM+ if const_expr(not is_ts):+ assert sA is not None, "sA must be provided when a_src is not TMEM"+ sA_layout = sA.layout if sA is not None else tCrA.layout+ sB_layout = sB.layout+ idesc: int = const_expr(sm100_desc.mma_op_to_idesc(op))+ if const_expr(not is_ts):+ sA_swizzle = parse_swizzle_from_pointer(sA.iterator)+ smem_desc_base_a: int = const_expr(+ sm100_desc.make_smem_desc_base(+ cute.recast_layout(128, op.a_dtype.width, sA_layout[0]),+ sA_swizzle,+ sm100_desc.Major.K+ if const_expr(op.a_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K)+ else sm100_desc.Major.MN,+ )+ )+ smem_desc_base_a_lo, smem_desc_a_hi = i64_to_i32x2(smem_desc_base_a)+ smem_desc_base_a_lo = const_expr(smem_desc_base_a_lo)+ smem_desc_a_hi = const_expr(smem_desc_a_hi)+ else:+ smem_desc_base_a = None+ smem_desc_base_a_lo, smem_desc_a_hi = None, None+ sB_swizzle = parse_swizzle_from_pointer(sB.iterator)+ smem_desc_base_b: int = const_expr(+ sm100_desc.make_smem_desc_base(+ cute.recast_layout(128, op.b_dtype.width, sB_layout[0]),+ sB_swizzle,+ sm100_desc.Major.K+ if const_expr(op.b_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K)+ else sm100_desc.Major.MN,+ )+ )+ smem_desc_base_b_lo, smem_desc_b_hi = i64_to_i32x2(smem_desc_base_b)+ smem_desc_base_b_lo = const_expr(smem_desc_base_b_lo)+ smem_desc_b_hi = const_expr(smem_desc_b_hi)+ tCrA_layout = (+ tCrA.layout+ if const_expr(not is_ts)+ else cute.recast_layout(32, tCrA.element_type.width, tCrA.layout)+ )+ offset_a = [cute.crd2idx((0, 0, k), tCrA_layout) for k in range(cute.size(tCrA.shape[2]))]+ offset_a_diff = [offset_a[k] - offset_a[k - 1] for k in range(1, cute.size(tCrA.shape[2]))]+ offset_b = [cute.crd2idx((0, 0, k), tCrB.layout) for k in range(cute.size(tCrB.shape[2]))]+ offset_b_diff = [offset_b[k] - offset_b[k - 1] for k in range(1, cute.size(tCrB.shape[2]))]+ # A K step's scales start at a byte offset into TMEM. PTX addresses take+ # 32-bit columns, so the column is that offset over four, and the byte+ # inside the column is the descriptor's scale-factor id. NVFP4's four+ # scales per step fill a column (id 0); MXFP4's two share one with the+ # next step, which reads them from byte 2.+ sf_bytes_a = [cute.crd2idx((0, 0, k), tScaleA.layout) for k in range(cute.size(tCrA.shape[2]))]+ sf_bytes_b = [cute.crd2idx((0, 0, k), tScaleB.layout) for k in range(cute.size(tCrB.shape[2]))]+ offset_sfa = [offset // 4 for offset in sf_bytes_a]+ offset_sfb = [offset // 4 for offset in sf_bytes_b]+ idesc_k = [+ idesc | (sf_bytes_a[k] % 4) << 29 | (sf_bytes_b[k] % 4) << 4+ for k in range(cute.size(tCrA.shape[2]))+ ]+ scale_vec = {16: "4X", 32: "2X"}[op.sf_vec_size]+ mma_kind = {16: "mxf4nvf4", 32: "mxf4"}[op.sf_vec_size]+ if const_expr(not is_ts):+ smem_desc_start_a_lo = Int32(+ smem_desc_base_a_lo | sm100_desc.make_smem_desc_start_addr(sA[None, None, 0].iterator)+ )+ # ) + sA_offset+ else:+ smem_desc_start_a_lo = None+ smem_desc_start_b_lo = Int32(+ smem_desc_base_b_lo | sm100_desc.make_smem_desc_start_addr(sB[None, None, 0].iterator)+ )+ pred_str = "p" if isinstance(zero_init, Boolean) else "0" if zero_init else "1"+ if const_expr(not is_ts):+ assert mbar_ptr is None, "mbar_ptr must be None when a_src is not TMEM"+ # Compute base addresses for scale factors+ scale_A_base_addr = tScaleA[None, None, 0].iterator.toint()+ scale_B_base_addr = tScaleB[None, None, 0].iterator.toint()+ mma_inst_str = f"tcgen05.mma.cta_group::1.kind::{mma_kind}.block_scale.scale_vec::{scale_vec}"+ llvm.inline_asm(+ None,+ [+ # acc.iterator.toint().ir_value(),+ Int32(cute.arch.make_warp_uniform(smem_desc_start_a_lo)).ir_value(),+ Int32(cute.arch.make_warp_uniform(smem_desc_start_b_lo)).ir_value(),+ Int32(not zero_init).ir_value(),+ Int32(cute.arch.make_warp_uniform(acc_tmem_addr)).ir_value(),+ Int32(cute.arch.make_warp_uniform(scale_A_base_addr)).ir_value(),+ Int32(cute.arch.make_warp_uniform(scale_B_base_addr)).ir_value(),+ ],+ "{\n\t"+ ".reg .pred leader_thread;\n\t"+ ".reg .pred p;\n\t"+ ".reg .b32 idesc;\n\t"+ ".reg .b32 tmem_acc;\n\t"+ ".reg .b32 tmem_scale_a;\n\t"+ ".reg .b32 tmem_scale_b;\n\t"+ ".reg .b32 smem_desc_a_lo_start, smem_desc_b_lo_start;\n\t"+ ".reg .b32 smem_desc_a_lo, smem_desc_b_lo;\n\t"+ ".reg .b32 smem_desc_a_hi, smem_desc_b_hi;\n\t"+ ".reg .b64 smem_desc_a, smem_desc_b;\n\t"+ "elect.sync _|leader_thread, -1;\n\t"+ f"mov.b32 idesc, {hex(idesc)};\n\t"+ # f"mov.b32 tmem_acc, {hex(acc_tmem_addr)};\n\t"+ f"mov.b32 tmem_acc, $3;\n\t"+ f"mov.b32 tmem_scale_a, $4;\n\t"+ f"mov.b32 tmem_scale_b, $5;\n\t"+ "mov.b32 smem_desc_a_lo_start, $0;\n\t"+ "mov.b32 smem_desc_b_lo_start, $1;\n\t"+ f"mov.b32 smem_desc_a_hi, {hex(smem_desc_a_hi)};\n\t"+ f"mov.b32 smem_desc_b_hi, {hex(smem_desc_b_hi)};\n\t"+ f"mov.b64 smem_desc_a, {{smem_desc_a_lo_start, smem_desc_a_hi}};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo_start, smem_desc_b_hi}};\n\t"+ "setp.ne.b32 p, $2, 0;\n\t"+ f"@leader_thread {mma_inst_str} [tmem_acc], smem_desc_a, smem_desc_b, idesc, [tmem_scale_a], [tmem_scale_b], {pred_str};\n\t"+ + "".join(+ (+ # f"add.u32 smem_desc_a_lo, smem_desc_a_lo, {hex(offset_a_diff[k - 1])};\n\t"+ # f"add.u32 smem_desc_b_lo, smem_desc_b_lo, {hex(offset_b_diff[k - 1])};\n\t"+ f"add.u32 smem_desc_a_lo, smem_desc_a_lo_start, {hex(offset_a[k])};\n\t"+ f"add.u32 smem_desc_b_lo, smem_desc_b_lo_start, {hex(offset_b[k])};\n\t"+ f"mov.b64 smem_desc_a, {{smem_desc_a_lo, smem_desc_a_hi}};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t"+ f"mov.b32 idesc, {hex(idesc_k[k])};\n\t"+ f"@leader_thread {mma_inst_str} [tmem_acc], smem_desc_a, smem_desc_b, idesc, [tmem_scale_a + {hex(offset_sfa[k])}], [tmem_scale_b + {hex(offset_sfb[k])}], 1;\n\t"+ )+ for k in range(1, cute.size(tCrA.shape[2]))+ )+ + "}\n",+ # "r,r,r",+ "r,r,r,r,r,r",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ else:+ # For TS gemm, somehow tCrA.iterator.toint() returns 0 no matter what, so we need to+ # explicitly pass in the tA_addr for correctness.+ tA_addr = tCrA[None, None, 0].iterator.toint() if tA_addr is None else tA_addr+ # Compute base addresses for scale factors similar to tA_addr+ scale_A_base_addr = tScaleA[None, None, 0].iterator.toint()+ scale_B_base_addr = tScaleB[None, None, 0].iterator.toint()+ mma_inst_str = f"tcgen05.mma.cta_group::1.kind::{mma_kind}.block_scale.scale_vec::{scale_vec}"+ input_args = [+ # Int32(cute.arch.make_warp_uniform(tCrA[None, None, 0].iterator.toint())).ir_value(),+ Int32(cute.arch.make_warp_uniform(tA_addr)).ir_value(),+ Int32(cute.arch.make_warp_uniform(smem_desc_start_b_lo)).ir_value(),+ Int32(not zero_init).ir_value(),+ Int32(cute.arch.make_warp_uniform(acc_tmem_addr)).ir_value(),+ Int32(cute.arch.make_warp_uniform(scale_A_base_addr)).ir_value(),+ Int32(cute.arch.make_warp_uniform(scale_B_base_addr)).ir_value(),+ ]+ if const_expr(mbar_ptr is not None):+ assert mbar_phase is not None, "mbar_phase must be provided when mbar_ptr is not None"+ input_args.append(mbar_ptr.toint().ir_value())+ input_args.append(Int32(mbar_phase).ir_value())+ mbar_wait_str = (+ ".reg .pred P1; \n\t"+ "LAB_WAIT: \n\t"+ "mbarrier.try_wait.parity.shared::cta.b64 P1, [$6], $7, 10000000; \n\t"+ "@P1 bra DONE; \n\t"+ "bra LAB_WAIT; \n\t"+ "DONE: \n\t"+ )+ else:+ mbar_wait_str = ""++ llvm.inline_asm(+ None,+ input_args,+ "{\n\t"+ ".reg .pred leader_thread;\n\t"+ ".reg .pred p;\n\t"+ ".reg .b32 idesc;\n\t"+ ".reg .b32 tmem_acc;\n\t"+ ".reg .b32 tmem_a;\n\t"+ ".reg .b32 smem_desc_b_lo_start;\n\t"+ ".reg .b32 smem_desc_b_lo;\n\t"+ ".reg .b32 smem_desc_b_hi;\n\t"+ ".reg .b64 smem_desc_b;\n\t"+ ".reg .b32 tmem_scale_a;\n\t"+ ".reg .b32 tmem_scale_b;\n\t"+ "elect.sync _|leader_thread, -1;\n\t"+ f"mov.b32 idesc, {hex(idesc)};\n\t"+ # f"mov.b32 tmem_acc, {hex(acc_tmem_addr)};\n\t"+ f"mov.b32 tmem_acc, $3;\n\t"+ f"mov.b32 tmem_a, $0;\n\t"+ f"mov.b32 tmem_scale_a, $4;\n\t"+ f"mov.b32 tmem_scale_b, $5;\n\t"+ f"mov.b32 smem_desc_b_lo_start, $1;\n\t"+ f"mov.b32 smem_desc_b_hi, {hex(smem_desc_b_hi)};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo_start, smem_desc_b_hi}};\n\t"+ "setp.ne.b32 p, $2, 0;\n\t"+ f"@leader_thread {mma_inst_str} [tmem_acc], [tmem_a], smem_desc_b, idesc, [tmem_scale_a], [tmem_scale_b], {pred_str};\n\t"+ + "".join(+ (+ f"add.u32 smem_desc_b_lo, smem_desc_b_lo_start, {hex(offset_b[k])};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t"+ f"mov.b32 idesc, {hex(idesc_k[k])};\n\t"+ f"@leader_thread {mma_inst_str} [tmem_acc], [tmem_a + {hex(offset_a[k])}], smem_desc_b, idesc, [tmem_scale_a + {hex(offset_sfa[k])}], [tmem_scale_b + {hex(offset_sfb[k])}], 1;\n\t"+ )+ for k in range(+ 1,+ cute.size(tCrA.shape[2])+ if const_expr(mbar_ptr is None)+ else (pre_mbar_tiles if const_expr(pre_mbar_tiles is not None) else cute.size(tCrA.shape[2]) // 4 * 3),+ )+ )+ + mbar_wait_str+ + (+ "".join(+ (+ f"add.u32 smem_desc_b_lo, smem_desc_b_lo_start, {hex(offset_b[k])};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t"+ f"mov.b32 idesc, {hex(idesc_k[k])};\n\t"+ f"@leader_thread {mma_inst_str} [tmem_acc], [tmem_a + {hex(offset_a[k])}], smem_desc_b, idesc, [tmem_scale_a + {hex(offset_sfa[k])}], [tmem_scale_b + {hex(offset_sfb[k])}], 1;\n\t"+ )+ for k in range(max(1, pre_mbar_tiles if const_expr(pre_mbar_tiles is not None) else cute.size(tCrA.shape[2]) // 4 * 3), cute.size(tCrA.shape[2]))+ )+ if const_expr(mbar_ptr is not None)+ else ""+ )+ + "}\n",+ "r,r,r,r,r,r" if const_expr(mbar_ptr is None) else "r,r,r,r,r,r,r,r",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+++def gemm_ptx_partial1(+ op: cute.nvgpu.tcgen05.mma.MmaOp,+ acc_tmem_addr: cutlass.Constexpr[int],+ tCrA: cute.Tensor,+ tCrB: cute.Tensor,+ sA_base_addr_for_desc: Int32,+ sA_addr_offset_for_desc: cutlass.Constexpr[int],+ sA_stage: Int32,+ sB_base_addr_for_desc: Int32,+ sB_addr_offset_for_desc: cutlass.Constexpr[int],+ sB_stage: Int32,+ sA_layout: Optional[cute.Layout],+ sB_layout: Optional[cute.Layout],+ sA_swizzle: Optional[cute.Swizzle],+ sB_swizzle: cute.Swizzle,+ zero_init: bool | Boolean = False,+) -> None:+ is_ts = op.a_src == cute.nvgpu.tcgen05.OperandSource.TMEM+ if const_expr(not is_ts):+ assert sA_layout is not None, "sA_layout must be provided when a_src is not TMEM"+ assert sA_swizzle is not None, "sA_swizzle must be provided when a_src is not TMEM"+ idesc: int = const_expr(sm100_desc.mma_op_to_idesc(op))+ if const_expr(not is_ts):+ smem_desc_base_a: int = const_expr(+ sm100_desc.make_smem_desc_base(+ cute.recast_layout(128, op.a_dtype.width, sA_layout[0]),+ sA_swizzle,+ sm100_desc.Major.K+ if const_expr(op.a_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K)+ else sm100_desc.Major.MN,+ )+ )+ smem_desc_base_a_lo, smem_desc_a_hi = i64_to_i32x2(smem_desc_base_a)+ smem_desc_base_a_lo = const_expr(smem_desc_base_a_lo)+ smem_desc_a_hi = const_expr(smem_desc_a_hi)+ else:+ smem_desc_base_a = None+ smem_desc_base_a_lo, smem_desc_a_hi = None, None+ smem_desc_base_b: int = const_expr(+ sm100_desc.make_smem_desc_base(+ cute.recast_layout(128, op.b_dtype.width, sB_layout[0]),+ sB_swizzle,+ sm100_desc.Major.K+ if const_expr(op.b_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K)+ else sm100_desc.Major.MN,+ )+ )+ smem_desc_base_b_lo, smem_desc_b_hi = i64_to_i32x2(smem_desc_base_b)+ smem_desc_base_b_lo = const_expr(smem_desc_base_b_lo)+ smem_desc_b_hi = const_expr(smem_desc_b_hi)+ mask = [Int32(0)] * 4++ if const_expr(not is_ts):+ offset_a = [+ (cute.crd2idx((0, 0, k), sA_layout) * op.a_dtype.width // 8) >> 4+ for k in range(cute.size(tCrA.shape[2]))+ ]+ else:+ offset_a = [+ cute.crd2idx((0, 0, k), sA_layout) * op.a_dtype.width // 32+ for k in range(cute.size(tCrA.shape[2]))+ ]+ offset_a_diff = [offset_a[k] - offset_a[k - 1] for k in range(1, cute.size(tCrA.shape[2]))]+ offset_b = [+ (cute.crd2idx((0, 0, k), sB_layout) * op.b_dtype.width // 8) >> 4+ for k in range(cute.size(tCrB.shape[2]))+ ]+ offset_b_diff = [offset_b[k] - offset_b[k - 1] for k in range(1, cute.size(tCrB.shape[2]))]++ if const_expr(not is_ts):+ # smem_desc_start_a_lo = Int32(smem_desc_base_a_lo | sm100_desc.make_smem_desc_start_addr(sA[None, None, 0].iterator))+ smem_desc_start_a_lo = const_expr(smem_desc_base_a_lo)+ else:+ smem_desc_start_a_lo = None+ # smem_desc_start_b_lo = Int32(smem_desc_base_b_lo | sm100_desc.make_smem_desc_start_addr(sB[None, None, 0].iterator))+ smem_desc_start_b_lo = const_expr(smem_desc_base_b_lo)+ pred_str = "p" if isinstance(zero_init, Boolean) else "0" if zero_init else "1"+ if const_expr(not is_ts):+ llvm.inline_asm(+ None,+ [+ # acc.iterator.toint().ir_value(),+ # Int32(cute.arch.make_warp_uniform(smem_desc_start_a_lo)).ir_value(),+ Int32(sA_base_addr_for_desc).ir_value(),+ Int32(sA_stage).ir_value(),+ # Int32(cute.arch.make_warp_uniform(smem_desc_start_b_lo)).ir_value(),+ Int32(sB_base_addr_for_desc).ir_value(),+ Int32(sB_stage).ir_value(),+ Int32(not zero_init).ir_value(),+ mask[0].ir_value(),+ mask[1].ir_value(),+ mask[2].ir_value(),+ mask[3].ir_value(),+ ],+ "{\n\t"+ ".reg .pred leader_thread;\n\t"+ ".reg .pred p;\n\t"+ ".reg .b32 idesc;\n\t"+ ".reg .b32 tmem_acc;\n\t"+ ".reg .b32 smem_desc_a_lo, smem_desc_b_lo;\n\t"+ ".reg .b32 smem_desc_a_hi, smem_desc_b_hi;\n\t"+ ".reg .b64 smem_desc_a, smem_desc_b;\n\t"+ "elect.sync _|leader_thread, -1;\n\t"+ f"mov.b32 idesc, {hex(idesc)};\n\t"+ f"mov.b32 tmem_acc, {hex(acc_tmem_addr)};\n\t"+ # "mov.b32 smem_desc_a_lo, $0;\n\t"+ # f"add.u32 smem_desc_a_lo, $0, {hex(smem_desc_start_a_lo)};\n\t"+ f"mad.lo.u32 smem_desc_a_lo, $1, {hex(sA_addr_offset_for_desc)}, $0;\n\t"+ # "mov.b32 smem_desc_b_lo, $2;\n\t"+ f"mad.lo.u32 smem_desc_b_lo, $3, {hex(sB_addr_offset_for_desc)}, $2;\n\t"+ f"mov.b32 smem_desc_a_hi, {hex(smem_desc_a_hi)};\n\t"+ f"mov.b32 smem_desc_b_hi, {hex(smem_desc_b_hi)};\n\t"+ f"mov.b64 smem_desc_a, {{smem_desc_a_lo, smem_desc_a_hi}};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t"+ "setp.ne.b32 p, $4, 0;\n\t"+ f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], smem_desc_a, smem_desc_b, idesc, {{$5, $6, $7, $8}}, {pred_str};\n\t"+ + "".join(+ (+ f"add.u32 smem_desc_a_lo, smem_desc_a_lo, {hex(offset_a_diff[k - 1])};\n\t"+ f"add.u32 smem_desc_b_lo, smem_desc_b_lo, {hex(offset_b_diff[k - 1])};\n\t"+ f"mov.b64 smem_desc_a, {{smem_desc_a_lo, smem_desc_a_hi}};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t"+ f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [tmem_acc], smem_desc_a, smem_desc_b, idesc, {{$5, $6, $7, $8}}, 1;\n\t"+ )+ for k in range(1, cute.size(tCrA.shape[2]))+ )+ + "}\n",+ "r,r,r,r,r,r,r,r,r",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ else:+ llvm.inline_asm(+ None,+ [+ # acc.iterator.toint().ir_value(),+ Int32(tCrA[None, None, 0].iterator.toint()).ir_value(),+ Int32(smem_desc_start_b_lo).ir_value(),+ Int32(not zero_init).ir_value(),+ mask[0].ir_value(),+ mask[1].ir_value(),+ mask[2].ir_value(),+ mask[3].ir_value(),+ ],+ "{\n\t"+ ".reg .pred leader_thread;\n\t"+ ".reg .pred p;\n\t"+ ".reg .b32 idesc;\n\t"+ ".reg .b32 tmem_a;\n\t"+ ".reg .b32 smem_desc_b_lo;\n\t"+ ".reg .b32 smem_desc_b_hi;\n\t"+ ".reg .b64 smem_desc_b;\n\t"+ "elect.sync _|leader_thread, -1;\n\t"+ f"mov.b32 idesc, {hex(idesc)};\n\t"+ f"mov.b32 tmem_a, $1;\n\t"+ f"mov.b32 smem_desc_b_lo, $2;\n\t"+ f"mov.b32 smem_desc_b_hi, {hex(smem_desc_b_hi)};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t"+ "setp.ne.b32 p, $3, 0;\n\t"+ f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [$0], [tmem_a], smem_desc_b, idesc, {{$4, $5, $6, $7}}, {pred_str};\n\t"+ + "".join(+ (+ f"add.u32 tmem_a, tmem_a, {hex(offset_a_diff[k - 1])};\n\t"+ f"add.u32 smem_desc_b_lo, smem_desc_b_lo, {hex(offset_b_diff[k - 1])};\n\t"+ f"mov.b64 smem_desc_b, {{smem_desc_b_lo, smem_desc_b_hi}};\n\t"+ f"@leader_thread tcgen05.mma.cta_group::1.kind::f16 [$0], [tmem_a], smem_desc_b, idesc, {{$4, $5, $6, $7}}, 1;\n\t"+ )+ for k in range(1, cute.size(tCrA.shape[2]))+ )+ + "}\n",+ "r,r,r,r,r,r,r,r",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+++def packed_float_to_e2m1x2(f_hi: Float32, f_lo: Float32, *, loc=None, ip=None) -> Int32:+ """Convert two FP32 values to a single byte of E2M1 nibbles.++ ``f_lo`` occupies the low nibble, which is where the even element of a+ K-major pair lives. The byte is replicated across the returned word; the+ caller masks it down. Used where a thread owns one element of the pair and+ the other arrives by shuffle, so the eight-wide packer would waste seven+ conversions.+ """+ out_uint32 = llvm.inline_asm(+ T.i32(),+ [Float32(f_hi).ir_value(loc=loc, ip=ip), Float32(f_lo).ir_value(loc=loc, ip=ip)],+ "{\n\t"+ ".reg .b8 byte0;\n\t"+ "cvt.rn.satfinite.e2m1x2.f32 byte0, $1, $2;\n\t"+ "mov.b32 $0, {byte0, byte0, byte0, byte0};\n\t"+ "}\n",+ "=r,f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ return Int32(out_uint32)+++# FP4 Quantization helper functions+def packed_float_to_ue4m3(f0: Float32, f1: Float32, f2: Float32, f3: Float32, *, loc=None, ip=None) -> Int32:+ """Convert 4 FP32 values to UE4M3 format packed in uint32_t"""+ out_uint32 = llvm.inline_asm(+ T.i32(),+ [Float32(f0).ir_value(loc=loc, ip=ip), Float32(f1).ir_value(loc=loc, ip=ip),+ Float32(f2).ir_value(loc=loc, ip=ip), Float32(f3).ir_value(loc=loc, ip=ip)],+ "{\n\t"+ ".reg .b16 lo;\n\t"+ ".reg .b16 hi;\n\t"+ "cvt.rn.satfinite.e4m3x2.f32 lo, $2, $1;\n\t"+ "cvt.rn.satfinite.e4m3x2.f32 hi, $4, $3;\n\t"+ "mov.b32 $0, {lo, hi};\n\t"+ "}\n",+ "=r,f,f,f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ return Int32(out_uint32)++def packed_float_to_e2m1(f0: Float32, f1: Float32, f2: Float32, f3: Float32,+ f4: Float32, f5: Float32, f6: Float32, f7: Float32,+ *, loc=None, ip=None) -> Int32:+ """Convert 8 FP32 values to E2M1 format packed in uint32_t"""+ out_uint32 = llvm.inline_asm(+ T.i32(),+ [Float32(f0).ir_value(loc=loc, ip=ip), Float32(f1).ir_value(loc=loc, ip=ip),+ Float32(f2).ir_value(loc=loc, ip=ip), Float32(f3).ir_value(loc=loc, ip=ip),+ Float32(f4).ir_value(loc=loc, ip=ip), Float32(f5).ir_value(loc=loc, ip=ip),+ Float32(f6).ir_value(loc=loc, ip=ip), Float32(f7).ir_value(loc=loc, ip=ip)],+ "{\n\t"+ ".reg .b8 byte0;\n\t"+ ".reg .b8 byte1;\n\t"+ ".reg .b8 byte2;\n\t"+ ".reg .b8 byte3;\n\t"+ "cvt.rn.satfinite.e2m1x2.f32 byte0, $2, $1;\n\t"+ "cvt.rn.satfinite.e2m1x2.f32 byte1, $4, $3;\n\t"+ "cvt.rn.satfinite.e2m1x2.f32 byte2, $6, $5;\n\t"+ "cvt.rn.satfinite.e2m1x2.f32 byte3, $8, $7;\n\t"+ "mov.b32 $0, {byte0, byte1, byte2, byte3};\n\t"+ "}\n",+ "=r,f,f,f,f,f,f,f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ return Int32(out_uint32)++def tcgen05_after_thread_sync():+ llvm.inline_asm(+ None,+ [],+ "tcgen05.fence::after_thread_sync;",+ "",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )\ No newline at end of filediff --git a/src/nvfp4_decode_kernel/_fa4/block_info.py b/src/nvfp4_decode_kernel/_fa4/block_info.pynew file mode 100644index 0000000..63ee59e--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/block_info.py@@ -0,0 +1,104 @@+# Copyright (c) 2025, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.+from typing import Tuple, Optional+from dataclasses import dataclass++import cutlass+import cutlass.cute as cute+from cutlass import Int32, const_expr++from .seqlen_info import SeqlenInfoQK+++@dataclass(frozen=True)+class BlockInfo:+ tile_m: cutlass.Constexpr[int]+ tile_n: cutlass.Constexpr[int]+ is_causal: cutlass.Constexpr[bool]+ is_local: cutlass.Constexpr[bool] = False+ is_split_kv: cutlass.Constexpr[bool] = False+ window_size_left: Optional[Int32] = None+ window_size_right: Optional[Int32] = None+ qhead_per_kvhead_packgqa: cutlass.Constexpr[int] = 1++ @cute.jit+ def get_n_block_min_max(+ self,+ seqlen_info: SeqlenInfoQK,+ m_block: Int32,+ split_idx: cutlass.Int32 = 0,+ num_splits: cutlass.Int32 = 1,+ ) -> Tuple[Int32, Int32]:+ n_block_max = cute.ceil_div(seqlen_info.seqlen_k, self.tile_n)+ if const_expr(self.is_causal or (self.is_local and self.window_size_right is not None)):+ m_idx_max = (m_block + 1) * self.tile_m+ if const_expr(self.qhead_per_kvhead_packgqa > 1):+ m_idx_max = cute.ceil_div(m_idx_max, self.qhead_per_kvhead_packgqa)+ n_idx = m_idx_max + seqlen_info.seqlen_k - seqlen_info.seqlen_q+ n_idx_right = n_idx if const_expr(self.is_causal) else n_idx + self.window_size_right+ n_block_max = min(n_block_max, cute.ceil_div(n_idx_right, self.tile_n))+ n_block_min = 0+ if const_expr(self.is_local and self.window_size_left is not None):+ m_idx_min = m_block * self.tile_m+ if const_expr(self.qhead_per_kvhead_packgqa > 1):+ m_idx_min = m_idx_min // self.qhead_per_kvhead_packgqa+ n_idx = m_idx_min + seqlen_info.seqlen_k - seqlen_info.seqlen_q+ n_idx_left = n_idx - self.window_size_left+ n_block_min = cutlass.max(n_idx_left // self.tile_n, 0)+ if cutlass.const_expr(self.is_split_kv):+ num_n_blocks_per_split = (+ cutlass.Int32(0)+ if n_block_max <= n_block_min+ else (n_block_max - n_block_min + num_splits - 1) // num_splits+ )+ n_block_min = n_block_min + split_idx * num_n_blocks_per_split+ n_block_max = cutlass.min(n_block_min + num_n_blocks_per_split, n_block_max)+ return n_block_min, n_block_max++ @cute.jit+ def get_m_block_min_max(self, seqlen_info: SeqlenInfoQK, n_block: Int32) -> Tuple[Int32, Int32]:+ m_block_max = cute.ceil_div(seqlen_info.seqlen_q, self.tile_m)+ m_block_min = 0+ if const_expr(self.is_causal):+ m_block_min = max(+ m_block_min,+ (n_block * self.tile_n + seqlen_info.seqlen_q - seqlen_info.seqlen_k)+ // self.tile_m,+ )+ return m_block_min, m_block_max++ @cute.jit+ def get_n_block_min_causal_local_mask(+ self,+ seqlen_info: SeqlenInfoQK,+ m_block: Int32,+ n_block_min: Int32,+ ) -> Int32:+ """If we have separate iterations with causal or local masking at the start, where do we stop"""+ m_idx_min = m_block * self.tile_m+ if const_expr(self.qhead_per_kvhead_packgqa > 1):+ m_idx_min = m_idx_min // self.qhead_per_kvhead_packgqa+ n_idx = m_idx_min + seqlen_info.seqlen_k - seqlen_info.seqlen_q+ n_idx_right = (+ n_idx+ if const_expr(not self.is_local or self.window_size_right is None)+ else n_idx + self.window_size_right+ )+ return cutlass.max(n_block_min, n_idx_right // self.tile_n)++ @cute.jit+ def get_n_block_min_before_local_mask(+ self,+ seqlen_info: SeqlenInfoQK,+ m_block: Int32,+ n_block_min: Int32,+ ) -> Int32:+ """If we have separate iterations with local masking at the end, where do we stop the non-masked iterations"""+ if const_expr(not self.is_local or self.window_size_left is None):+ return n_block_min+ else:+ m_idx_max = (m_block + 1) * self.tile_m+ if const_expr(self.qhead_per_kvhead_packgqa > 1):+ m_idx_max = cute.ceil_div(m_idx_max, self.qhead_per_kvhead_packgqa)+ n_idx = m_idx_max + seqlen_info.seqlen_k - seqlen_info.seqlen_q+ n_idx_left = n_idx - self.window_size_left+ return cutlass.max(n_block_min, cute.ceil_div(n_idx_left, self.tile_n))diff --git a/src/nvfp4_decode_kernel/_fa4/block_scaled_layout.py b/src/nvfp4_decode_kernel/_fa4/block_scaled_layout.pynew file mode 100644index 0000000..8b53c14--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/block_scaled_layout.py@@ -0,0 +1,290 @@+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: LicenseRef-NvidiaProprietary+#+# Use of this software is governed by the terms and conditions of the+# NVIDIA End User License Agreement (EULA), available at:+# https://docs.nvidia.com/cutlass/media/docs/pythonDSL/license.html+#+# Any use, reproduction, disclosure, or distribution of this software+# and related documentation outside the scope permitted by the EULA+# is strictly prohibited.++from dataclasses import dataclass, field++from cutlass.cutlass_dsl import dsl_user_op++import cutlass.cute as cute+from cutlass.cute.nvgpu.tcgen05 import OperandMajorMode++import cutlass._mlir.dialects.cute as _cute_ir+import cutlass._mlir.dialects.cute_nvgpu as _cute_nvgpu_ir+++@dataclass(frozen=True)+class BlockScaledBasicChunk:+ """+ The basic scale factor atom layout decided by tcgen05 BlockScaled MMA Ops.++ This class represents the fixed layout pattern for scale factors used in+ tcgen05 BlockScaled MMA Ops. The layout is determined by the+ instruction specification and cannot be modified.+ See `PTX documentation <https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-scale-factor-a-layout-1x>`.+ """++ sf_vec_size: int+ major_mode: OperandMajorMode = OperandMajorMode.K+ _layout: cute.Layout = field(init=False, repr=False)++ def __post_init__(self) -> None:+ if self.major_mode == OperandMajorMode.K:+ # K-major layout: (AtomMN, AtomK)+ atom_shape = ((32, 4), (self.sf_vec_size, 4))+ atom_stride = ((16, 4), (0, 1))+ else:+ # MN-major layout: (AtomK, AtomMN)+ atom_shape = ((self.sf_vec_size, 4), (32, 4))+ atom_stride = ((0, 1), (16, 4))++ object.__setattr__(+ self, "_layout", cute.make_layout(atom_shape, stride=atom_stride)+ )++ @property+ def layout(self) -> cute.Layout:+ """+ Get the layout for this block scaled chunk.++ :return: The layout representing the scale factor atom+ :rtype: cute.Layout+ """+ return self._layout+++@dsl_user_op+def tile_atom_to_shape_SF(+ Shape: cute.Shape,+ sf_vec_size: int,+ *,+ loc=None,+ ip=None,+) -> cute.Layout:+ """+ A helper function to get dynamic SFA/SFB layout by filling dynamic A/B shape to the scale factor atom layout.++ :param Shape: The shape of the A/B tensor+ :param sf_vec_size: Scale factor vector size++ :return: The layout of the SFA/SFB tensor+ :rtype: cute.Layout+ """+ # ((Atom_MN, Rest_MN),(Atom_K, Rest_K),RestL)+ sf_layout = cute.tile_to_shape(+ BlockScaledBasicChunk(sf_vec_size).layout, Shape, (2, 1, 3)+ )+ return sf_layout+++@dsl_user_op+def make_smem_layout_sfa(+ tiled_mma: cute.TiledMma,+ mma_tiler_mnk: cute.Tile,+ sf_vec_size: int,+ num_stages: int,+ *,+ loc=None,+ ip=None,+ mma_tile_inst_k=4,+) -> cute.Layout:+ """+ Make smem layout for SFA based on:++ 1. BlockScaledBasicChunk+ 2. MMA tiler shape+ 3. Scale factor vector size+ 4. Number of stages++ :param tiled_mma: The tiled MMA+ :type tiled_mma: cute.TiledMma+ :param mma_tiler_mnk: The mma tiler shape+ :type mma_tiler_mnk: cute.Tile+ :param sf_vec_size: The scale factor vector size+ :type sf_vec_size: int+ :param num_stages: The number of stages+ :type num_stages: int++ :return: Smem layout for SFA+ :rtype: cute.Layout+ """+ # (CTA_Tile_Shape_M, MMA_Tile_Shape_K)+ sfa_tile_shape = (+ mma_tiler_mnk[0] // cute.size(tiled_mma.thr_id.shape),+ mma_tiler_mnk[2],+ )++ # ((Atom_M, Rest_M),(Atom_K, Rest_K))+ smem_layout = cute.tile_to_shape(+ BlockScaledBasicChunk(sf_vec_size).layout,+ sfa_tile_shape,+ (2, 1),+ )++ # (CTA_Tile_Shape_M, MMA_Inst_Shape_K)+ sfa_tile_shape = cute.shape_div(sfa_tile_shape, (1, mma_tile_inst_k))+ # ((Atom_Inst_M, Atom_Inst_K), MMA_M, MMA_K))+ smem_layout = cute.tiled_divide(smem_layout, sfa_tile_shape)++ atom_m = 128+ tiler_inst = ((atom_m, sf_vec_size),)+ # (((Atom_Inst_M, Rest_M),(Atom_Inst_K, Rest_K)), MMA_M, MMA_K)+ smem_layout = cute.logical_divide(smem_layout, tiler_inst)++ # (((Atom_Inst_M, Rest_M),(Atom_Inst_K, Rest_K)), MMA_M, MMA_K, STAGE)+ sfa_smem_layout_staged = cute.append(+ smem_layout,+ cute.make_layout(+ num_stages, stride=cute.cosize(cute.filter_zeros(smem_layout))+ ),+ )++ return sfa_smem_layout_staged+++@dsl_user_op+def make_smem_layout_sfb(+ tiled_mma: cute.TiledMma,+ mma_tiler_mnk: cute.Tile,+ sf_vec_size: int,+ num_stages: int,+ *,+ loc=None,+ ip=None,+ mma_tile_inst_k=4,+ atom_n=128,+) -> cute.Layout:+ """+ Make smem layout for SFB based on:++ 1. BlockScaledBasicChunk+ 2. MMA tiler shape+ 3. Scale factor vector size+ 4. Number of stages++ :param tiled_mma: The tiled MMA+ :type tiled_mma: cute.TiledMma+ :param mma_tiler_mnk: The mma tiler shape+ :type mma_tiler_mnk: cute.Tile+ :param sf_vec_size: The scale factor vector size+ :type sf_vec_size: int+ :param num_stages: The number of stages+ :type num_stages: int++ :return: Smem layout for SFA+ :rtype: cute.Layout+ """+ # (Round_Up(CTA_Tile_Shape_N, 128), MMA_Tile_Shape_K)+ sfb_tile_shape = (+ cute.round_up(mma_tiler_mnk[1], 128),+ mma_tiler_mnk[2],+ )++ # ((Atom_N, Rest_N),(Atom_K, Rest_K))+ smem_layout = cute.tile_to_shape(+ BlockScaledBasicChunk(sf_vec_size).layout,+ sfb_tile_shape,+ (2, 1),+ )++ # (CTA_Tile_Shape_N, MMA_Inst_Shape_K)+ sfb_tile_shape = cute.shape_div(sfb_tile_shape, (1, mma_tile_inst_k))+ # ((Atom_Inst_N, Atom_Inst_K), MMA_N, MMA_K)+ smem_layout = cute.tiled_divide(smem_layout, sfb_tile_shape)+ atom_n = 128+ tiler_inst = ((atom_n, sf_vec_size),)+ # (((Atom_Inst_M, Rest_M),(Atom_Inst_K, Rest_K)), MMA_M, MMA_K)+ smem_layout = cute.logical_divide(smem_layout, tiler_inst)++ # (((Atom_Inst_M, Rest_M),(Atom_Inst_K, Rest_K)), MMA_M, MMA_K, STAGE)+ sfb_smem_layout_staged = cute.append(+ smem_layout,+ cute.make_layout(+ num_stages, stride=cute.cosize(cute.filter_zeros(smem_layout))+ ),+ )++ return sfb_smem_layout_staged+++@dsl_user_op+def make_tmem_layout_sfa(+ tiled_mma: cute.TiledMma,+ mma_tiler_mnk: cute.Tile,+ sf_vec_size: int,+ smem_layout: cute.Layout,+ *,+ loc=None,+ ip=None,+) -> cute.Layout:+ """Make tmem layout for SFA based on:++ 1. SFA smem layout per stage+ 2. Cta tile shape m+ 3. tiled MMA atom thr size+ 4. Scale factor vector size++ :param tiled_mma: The tiled MMA+ :type tiled_mma: cute.TiledMma+ :param mma_tiler_mnk: The mma tiler shape+ :type mma_tiler_mnk: cute.Tile+ :param sf_vec_size: The scale factor vector size+ :type sf_vec_size: int+ :param smem_layout: The smem layout of SFA per stage+ :type smem_layout: cute.Layout++ :return: TMEM layout for SFA+ :rtype: cute.Layout+ """+ atom_thr_size = cute.size(tiled_mma.thr_id.shape, loc=loc, ip=ip)+ cta_tile_shape_m = mma_tiler_mnk[0] // atom_thr_size++ sfa_layout_ty = _cute_nvgpu_ir.make_tmem_layout_sfa(+ smem_layout, cta_tile_shape_m, atom_thr_size, sf_vec_size+ )+ return _cute_ir.static(sfa_layout_ty, loc=loc, ip=ip)+++@dsl_user_op+def make_tmem_layout_sfb(+ tiled_mma: cute.TiledMma,+ mma_tiler_mnk: cute.Tile,+ sf_vec_size: int,+ smem_layout: cute.Layout,+ *,+ loc=None,+ ip=None,+) -> cute.Layout:+ """Make tmem layout for SFB based on:++ 1. SFB smem layout per stage+ 2. Cta tile shape m+ 3. tiled MMA atom thr size+ 4. Scale factor vector size++ :param tiled_mma: The tiled MMA+ :type tiled_mma: cute.TiledMma+ :param mma_tiler_mnk: The mma tiler shape+ :type mma_tiler_mnk: cute.Tile+ :param sf_vec_size: The scale factor vector size+ :type sf_vec_size: int+ :param smem_layout: The smem layout of SFB per stage+ :type smem_layout: cute.Layout++ :return: TMEM layout for SFB+ :rtype: cute.Layout+ """+ atom_thr_size = cute.size(tiled_mma.thr_id.shape, loc=loc, ip=ip)+ cta_tile_shape_m = mma_tiler_mnk[0] // atom_thr_size++ sfb_layout_ty = _cute_nvgpu_ir.make_tmem_layout_sfb(+ smem_layout, cta_tile_shape_m, atom_thr_size, sf_vec_size+ )+ return _cute_ir.static(sfb_layout_ty, loc=loc, ip=ip)diff --git a/src/nvfp4_decode_kernel/_fa4/block_sparse_utils.py b/src/nvfp4_decode_kernel/_fa4/block_sparse_utils.pynew file mode 100644index 0000000..c7b59f7--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/block_sparse_utils.py@@ -0,0 +1,805 @@+"""+Block-sparse runtime utilities for CUTE DSL kernels.++This module contains runtime execution functions for block-sparse attention kernels.+These utilities are used by CUTE DSL kernels to produce and consume block-sparse loads.+"""++from typing import Callable, Optional+from functools import partial+import math+import cutlass+import cutlass.cute as cute+from cutlass import Float32, Int32, const_expr++# Import data structures from block_sparsity+from .block_sparsity import BlockSparseTensors+from . import utils+++def load_block_list(+ block_indices: cute.Tensor,+ block_count,+ load_q_with_first: cutlass.Constexpr,+ first_block_preloaded: cutlass.Constexpr,+ kv_producer_state,+ load_Q,+ load_K,+ load_V,+ pipeline_k,+ pipeline_v,+ use_tma_q: cutlass.Constexpr,+ tma_q_bytes: cutlass.Constexpr,+ intra_wg_overlap: cutlass.Constexpr,+):+ """Iterate over the sparse blocks and load K, V (and Q) into the pipeline.+ for the intra_wg_overlap case, we overlap the loads of K and V. And this+ means we need to pipeline the last V load from the partial block case,+ with the loads for the full blocks. Set first_block_preloaded when the+ caller has already issued the first K load for the list.++ Note:+ we iterate along the block_n indices in reverse.++ Returns:+ Updated kv_producer_state after processing the block list.++ """+ if block_count > 0:+ if const_expr(not intra_wg_overlap):+ # Peel first iteration: the first block may need to load Q alongside K,+ # Parameters are already Constexpr, so no need to wrap in const_expr()+ n_block_first = block_indices[block_count - 1]+ extra_tx = tma_q_bytes if const_expr(load_q_with_first) and const_expr(use_tma_q) else 0+ pipeline_k.producer_acquire(kv_producer_state, extra_tx_count=extra_tx)++ if const_expr(load_q_with_first and use_tma_q):+ load_Q(tma_bar_ptr=pipeline_k.producer_get_barrier(kv_producer_state))++ load_K(src_idx=n_block_first, producer_state=kv_producer_state)+ pipeline_v.producer_acquire(kv_producer_state)+ load_V(src_idx=n_block_first, producer_state=kv_producer_state)+ kv_producer_state.advance()++ for offset in cutlass.range(1, block_count):+ n_block = block_indices[block_count - 1 - offset]+ pipeline_k.producer_acquire(kv_producer_state)+ load_K(src_idx=n_block, producer_state=kv_producer_state)+ pipeline_v.producer_acquire(kv_producer_state)+ load_V(src_idx=n_block, producer_state=kv_producer_state)+ kv_producer_state.advance()+ else:+ n_block_first = block_indices[block_count - 1]+ if const_expr(not first_block_preloaded):+ extra_tx = (+ tma_q_bytes if const_expr(load_q_with_first) and const_expr(use_tma_q) else 0+ )+ pipeline_k.producer_acquire(kv_producer_state, extra_tx_count=extra_tx)++ if const_expr(load_q_with_first and use_tma_q):+ load_Q(tma_bar_ptr=pipeline_k.producer_get_barrier(kv_producer_state))++ load_K(src_idx=n_block_first, producer_state=kv_producer_state)++ for idx in cutlass.range(block_count - 1, unroll=1):+ n_block_prev = block_indices[block_count - 1 - idx]+ n_block = block_indices[block_count - 2 - idx]+ kv_producer_state_prev = kv_producer_state.clone()+ kv_producer_state.advance()+ pipeline_k.producer_acquire(kv_producer_state)+ load_K(src_idx=n_block, producer_state=kv_producer_state)+ pipeline_v.producer_acquire(kv_producer_state_prev)+ load_V(src_idx=n_block_prev, producer_state=kv_producer_state_prev)++ return kv_producer_state+++def finish_overlap_v_load(+ block_indices: cute.Tensor,+ block_count,+ load_V,+ pipeline_v,+ kv_producer_state,+):+ """Load the final V block after overlapped K/V loads."""+ if block_count > 0:+ n_block_last = block_indices[0]+ pipeline_v.producer_acquire(kv_producer_state)+ load_V(src_idx=n_block_last, producer_state=kv_producer_state)+ kv_producer_state.advance()++ return kv_producer_state+++def produce_block_sparse_loads(+ blocksparse_tensors: BlockSparseTensors,+ batch_idx,+ head_idx,+ m_block,+ kv_producer_state,+ load_Q,+ load_K,+ load_V,+ pipeline_k,+ pipeline_v,+ use_tma_q: cutlass.Constexpr,+ tma_q_bytes: cutlass.Constexpr,+ intra_wg_overlap: cutlass.Constexpr,+):+ """Iterate over the mask and full block lists for a single tile.++ The masked (partial) list may leave the last V load pending when intra-warp-group+ overlap is enabled. The first full block must consume that pending V while+ issuing its own K load on the next pipeline stage.++ In the intra-wg-overlap path, the last masked block leaves its V copy in flight+ while we advance the producer state to start the next full K. Either the full list+ overlaps that pending V load, or, if no full blocks exist, we explicitly drain it.++ """++ mask_block_cnt, mask_block_idx, full_block_cnt, full_block_idx = blocksparse_tensors++ curr_mask_block_cnt = mask_block_cnt[batch_idx, head_idx, m_block]+ curr_mask_block_idx = mask_block_idx[batch_idx, head_idx, m_block, None]++ if const_expr(full_block_cnt is not None):+ curr_full_block_cnt = full_block_cnt[batch_idx, head_idx, m_block]+ curr_full_block_idx = full_block_idx[batch_idx, head_idx, m_block, None]+ else:+ curr_full_block_cnt = Int32(0)+ curr_full_block_idx = None++ mask_empty = curr_mask_block_cnt == 0+ full_empty = curr_full_block_cnt == 0++ if mask_empty:+ # No masked blocks: the full list owns the initial Q+K load.+ kv_producer_state = load_block_list(+ curr_full_block_idx,+ curr_full_block_cnt,+ load_q_with_first=True,+ first_block_preloaded=False,+ kv_producer_state=kv_producer_state,+ load_Q=load_Q,+ load_K=load_K,+ load_V=load_V,+ pipeline_k=pipeline_k,+ pipeline_v=pipeline_v,+ use_tma_q=use_tma_q,+ tma_q_bytes=tma_q_bytes,+ intra_wg_overlap=intra_wg_overlap,+ )++ if const_expr(intra_wg_overlap) and curr_full_block_cnt > 0:+ kv_producer_state = finish_overlap_v_load(+ curr_full_block_idx,+ curr_full_block_cnt,+ load_V,+ pipeline_v,+ kv_producer_state,+ )+ else:+ # Masked blocks present: load Q together with the first masked K so consumers can+ # start immediately. When overlap is disabled this fully drains the list.+ kv_producer_state = load_block_list(+ curr_mask_block_idx,+ curr_mask_block_cnt,+ load_q_with_first=True,+ first_block_preloaded=False,+ kv_producer_state=kv_producer_state,+ load_Q=load_Q,+ load_K=load_K,+ load_V=load_V,+ pipeline_k=pipeline_k,+ pipeline_v=pipeline_v,+ use_tma_q=use_tma_q,+ tma_q_bytes=tma_q_bytes,+ intra_wg_overlap=intra_wg_overlap,+ )++ if full_empty:+ if const_expr(intra_wg_overlap):+ kv_producer_state = finish_overlap_v_load(+ curr_mask_block_idx,+ curr_mask_block_cnt,+ load_V,+ pipeline_v,+ kv_producer_state,+ )+ else:+ if const_expr(intra_wg_overlap):+ # Bridge the masked list to the full list by overlapping the pending masked V+ # with the first full K load.+ n_block_mask_last = curr_mask_block_idx[0]+ n_block_full_first = curr_full_block_idx[curr_full_block_cnt - 1]+ kv_producer_state_prev = kv_producer_state.clone()+ kv_producer_state.advance()+ pipeline_k.producer_acquire(kv_producer_state)+ load_K(src_idx=n_block_full_first, producer_state=kv_producer_state)+ pipeline_v.producer_acquire(kv_producer_state_prev)+ load_V(src_idx=n_block_mask_last, producer_state=kv_producer_state_prev)++ kv_producer_state = load_block_list(+ curr_full_block_idx,+ curr_full_block_cnt,+ load_q_with_first=False,+ first_block_preloaded=True,+ kv_producer_state=kv_producer_state,+ load_Q=load_Q,+ load_K=load_K,+ load_V=load_V,+ pipeline_k=pipeline_k,+ pipeline_v=pipeline_v,+ use_tma_q=use_tma_q,+ tma_q_bytes=tma_q_bytes,+ intra_wg_overlap=intra_wg_overlap,+ )++ kv_producer_state = finish_overlap_v_load(+ curr_full_block_idx,+ curr_full_block_cnt,+ load_V,+ pipeline_v,+ kv_producer_state,+ )+ else:+ # Non-overlap path with both lists: run the full list normally (skipping the Q+ # reload because the masked list already issued it).+ kv_producer_state = load_block_list(+ curr_full_block_idx,+ curr_full_block_cnt,+ load_q_with_first=False,+ first_block_preloaded=False,+ kv_producer_state=kv_producer_state,+ load_Q=load_Q,+ load_K=load_K,+ load_V=load_V,+ pipeline_k=pipeline_k,+ pipeline_v=pipeline_v,+ use_tma_q=use_tma_q,+ tma_q_bytes=tma_q_bytes,+ intra_wg_overlap=intra_wg_overlap,+ )++ return kv_producer_state+++def consume_block_sparse_loads(+ blocksparse_tensors: BlockSparseTensors,+ batch_idx,+ head_idx,+ m_block,+ kv_consumer_state,+ mma_pv_fn,+ mma_one_n_block,+ process_first_half_block,+ process_last_half_block,+ mask_fn,+ score_mod_fn,+ O_should_accumulate,+ mask_mod,+ intra_wg_overlap: cutlass.Constexpr,+ warp_scheduler_barrier_sync: Callable,+ warp_scheduler_barrier_arrive: Callable,+):+ """Consume the mask and full block lists for a single tile on the consumer side.++ Mirrors `produce_block_sparse_loads` so that the consumer pipeline+ """++ mask_block_cnt, mask_block_idx, full_block_cnt, full_block_idx = blocksparse_tensors++ curr_mask_block_cnt = mask_block_cnt[batch_idx, head_idx, m_block]+ curr_mask_block_idx = mask_block_idx[batch_idx, head_idx, m_block, None]+ curr_full_block_cnt = full_block_cnt[batch_idx, head_idx, m_block]+ curr_full_block_idx = full_block_idx[batch_idx, head_idx, m_block, None]++ processed_any = curr_mask_block_cnt + curr_full_block_cnt > 0++ if const_expr(not intra_wg_overlap):+ if curr_mask_block_cnt > 0:+ mask_n_block = curr_mask_block_idx[curr_mask_block_cnt - 1]+ warp_scheduler_barrier_sync()+ kv_consumer_state = mma_one_n_block(+ kv_consumer_state,+ n_block=mask_n_block,+ mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate),+ mask_fn=partial(mask_fn, mask_mod=mask_mod, mask_seqlen=True),+ is_first_n_block=True,+ )+ O_should_accumulate = True+ for i in cutlass.range(1, curr_mask_block_cnt):+ mask_n_block = curr_mask_block_idx[curr_mask_block_cnt - 1 - i]+ kv_consumer_state = mma_one_n_block(+ kv_consumer_state,+ n_block=mask_n_block,+ mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate),+ mask_fn=partial(mask_fn, mask_mod=mask_mod, mask_seqlen=False),+ is_first_n_block=False,+ )+ O_should_accumulate = True+ if curr_full_block_cnt == 0:+ warp_scheduler_barrier_arrive()++ if curr_full_block_cnt > 0:+ full_n_block = curr_full_block_idx[curr_full_block_cnt - 1]+ if curr_mask_block_cnt == 0:+ warp_scheduler_barrier_sync()+ kv_consumer_state = mma_one_n_block(+ kv_consumer_state,+ n_block=full_n_block,+ mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate),+ mask_fn=partial(mask_fn, mask_seqlen=True),+ is_first_n_block=True,+ )+ O_should_accumulate = True+ for i in cutlass.range(1, curr_full_block_cnt):+ full_n_block = curr_full_block_idx[curr_full_block_cnt - 1 - i]+ kv_consumer_state = mma_one_n_block(+ kv_consumer_state,+ n_block=full_n_block,+ mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate),+ mask_fn=partial(mask_fn, mask_seqlen=False),+ is_first_n_block=False,+ )+ O_should_accumulate = True+ else:+ kv_consumer_state = mma_one_n_block(+ kv_consumer_state,+ n_block=full_n_block,+ mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate),+ mask_fn=partial(mask_fn, mask_mod=None, mask_seqlen=True),+ is_first_n_block=False,+ )+ O_should_accumulate = True+ for i in cutlass.range(1, curr_full_block_cnt):+ full_n_block = curr_full_block_idx[curr_full_block_cnt - 1 - i]+ kv_consumer_state = mma_one_n_block(+ kv_consumer_state,+ n_block=full_n_block,+ mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate),+ mask_fn=partial(mask_fn, mask_mod=None, mask_seqlen=False),+ is_first_n_block=False,+ )+ O_should_accumulate = True+ warp_scheduler_barrier_arrive()+ else:+ if curr_mask_block_cnt > 0:+ mask_n_block = curr_mask_block_idx[curr_mask_block_cnt - 1]+ kv_consumer_state = process_first_half_block(+ n_block=mask_n_block,+ kv_consumer_state=kv_consumer_state,+ mask_fn=partial(mask_fn, mask_mod=mask_mod),+ score_mod_fn=score_mod_fn,+ is_first_block=True,+ )+ for i in cutlass.range(1, curr_mask_block_cnt):+ mask_n_block = curr_mask_block_idx[curr_mask_block_cnt - 1 - i]+ kv_consumer_state = mma_one_n_block(+ kv_consumer_state,+ n_block=mask_n_block,+ mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate),+ mask_fn=partial(mask_fn, mask_mod=mask_mod, mask_seqlen=False),+ )+ O_should_accumulate = True++ if curr_full_block_cnt > 0:+ full_n_block = curr_full_block_idx[curr_full_block_cnt - 1]+ if curr_mask_block_cnt == 0:+ kv_consumer_state = process_first_half_block(+ n_block=full_n_block,+ kv_consumer_state=kv_consumer_state,+ mask_fn=partial(mask_fn, mask_mod=None),+ score_mod_fn=score_mod_fn,+ is_first_block=True,+ )+ else:+ kv_consumer_state = mma_one_n_block(+ kv_consumer_state,+ n_block=full_n_block,+ mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate),+ mask_fn=partial(mask_fn, mask_mod=None, mask_seqlen=True),+ )+ O_should_accumulate = True+ for i in cutlass.range(1, curr_full_block_cnt):+ full_n_block = curr_full_block_idx[curr_full_block_cnt - 1 - i]+ kv_consumer_state = mma_one_n_block(+ kv_consumer_state,+ n_block=full_n_block,+ mma_pv_fn=partial(mma_pv_fn, zero_init=not O_should_accumulate),+ mask_fn=partial(mask_fn, mask_mod=None, mask_seqlen=False),+ )+ O_should_accumulate = True++ if curr_mask_block_cnt + curr_full_block_cnt > 0:+ kv_consumer_state = process_last_half_block(+ kv_consumer_state=kv_consumer_state,+ zero_init=not O_should_accumulate,+ )+ O_should_accumulate = True++ return kv_consumer_state, O_should_accumulate, processed_any+++def load_block_list_sm100(+ block_indices: cute.Tensor,+ block_count,+ load_q_with_first: cutlass.Constexpr,+ m_block,+ q_stage: cutlass.Constexpr,+ kv_producer_state,+ load_Q,+ load_K,+ load_V,+ pipeline_kv,+):+ """SM100 version of load_block_list (no intra_wg_overlap, no extra_tx_count)."""+ if block_count > 0:+ # First iteration: load Q alongside K if requested+ n_block_first = block_indices[block_count - 1]++ if const_expr(load_q_with_first):+ # SM100 loads Q0 and optionally Q1+ load_Q(block=q_stage * m_block + 0, stage=0)+ if const_expr(q_stage == 2):+ load_Q(block=q_stage * m_block + 1, stage=1)++ # SM100 doesn't use producer_acquire for pipeline_kv in load path+ # The pipeline barriers are handled inside load_KV+ load_K(block=n_block_first, producer_state=kv_producer_state, page_idx=None)+ kv_producer_state.advance()+ load_V(block=n_block_first, producer_state=kv_producer_state, page_idx=None)+ kv_producer_state.advance()++ # Remaining blocks+ for offset in cutlass.range(1, block_count):+ n_block = block_indices[block_count - 1 - offset]+ load_K(block=n_block, producer_state=kv_producer_state, page_idx=None)+ kv_producer_state.advance()+ load_V(block=n_block, producer_state=kv_producer_state, page_idx=None)+ kv_producer_state.advance()++ return kv_producer_state+++# SM100-specific tile processor using SM100 helpers+def produce_block_sparse_loads_sm100(+ blocksparse_tensors: BlockSparseTensors,+ batch_idx,+ head_idx,+ m_block,+ kv_producer_state,+ load_Q,+ load_K,+ load_V,+ pipeline_kv,+ q_stage: cutlass.Constexpr,+ q_producer_phase: Int32,+):+ """SM100 entry point for sparse block iteration.++ SM100 uses PipelineTmaUmma which doesn't support extra_tx_count, so we use+ simplified block processing that just calls producer_acquire without extras.+ """+ mask_block_cnt, mask_block_idx, full_block_cnt, full_block_idx = blocksparse_tensors++ curr_mask_block_cnt = mask_block_cnt[batch_idx, head_idx, m_block]+ curr_mask_block_idx = mask_block_idx[batch_idx, head_idx, m_block, None]++ if const_expr(full_block_cnt is not None):+ curr_full_block_cnt = full_block_cnt[batch_idx, head_idx, m_block]+ curr_full_block_idx = full_block_idx[batch_idx, head_idx, m_block, None]+ else:+ curr_full_block_cnt = Int32(0)+ curr_full_block_idx = None++ mask_empty = curr_mask_block_cnt == 0+ full_empty = curr_full_block_cnt == 0++ q_phase_flipped = False++ if mask_empty:+ # No masked blocks: process full list with Q loading+ kv_producer_state = load_block_list_sm100(+ curr_full_block_idx,+ curr_full_block_cnt,+ load_q_with_first=True,+ m_block=m_block,+ q_stage=q_stage,+ kv_producer_state=kv_producer_state,+ load_Q=load_Q,+ load_K=load_K,+ load_V=load_V,+ pipeline_kv=pipeline_kv,+ )+ q_phase_flipped = not full_empty+ else:+ # Process masked blocks with Q loading+ kv_producer_state = load_block_list_sm100(+ curr_mask_block_idx,+ curr_mask_block_cnt,+ load_q_with_first=True,+ m_block=m_block,+ q_stage=q_stage,+ kv_producer_state=kv_producer_state,+ load_Q=load_Q,+ load_K=load_K,+ load_V=load_V,+ pipeline_kv=pipeline_kv,+ )+ q_phase_flipped = True++ if not full_empty:+ # Process full blocks without Q loading+ kv_producer_state = load_block_list_sm100(+ curr_full_block_idx,+ curr_full_block_cnt,+ load_q_with_first=False,+ m_block=m_block,+ q_stage=q_stage,+ kv_producer_state=kv_producer_state,+ load_Q=load_Q,+ load_K=load_K,+ load_V=load_V,+ pipeline_kv=pipeline_kv,+ )++ if q_phase_flipped:+ q_producer_phase ^= 1++ return kv_producer_state, q_producer_phase+++def get_total_block_count(+ blocksparse_tensors: BlockSparseTensors,+ batch_idx,+ head_idx,+ m_block,+):+ mask_block_cnt, mask_block_idx, full_block_cnt, full_block_idx = blocksparse_tensors+ if const_expr(full_block_cnt is not None):+ return (+ mask_block_cnt[batch_idx, head_idx, m_block]+ + full_block_cnt[batch_idx, head_idx, m_block]+ )+ else:+ return mask_block_cnt[batch_idx, head_idx, m_block]+++def handle_block_sparse_empty_tile_correction_sm100(+ tidx: Int32,+ q_stage: cutlass.Constexpr,+ m_block_size: cutlass.Constexpr,+ qhead_per_kvhead,+ pack_gqa: cutlass.Constexpr,+ is_split_kv: cutlass.Constexpr,+ learnable_sink,+ mLSE,+ seqlen,+ m_block: Int32,+ head_idx: Int32,+ batch_idx: Int32,+ split_idx: Int32,+ sScale: cute.Tensor,+ stats: list,+ correction_epilogue: Callable,+ thr_mma_pv: cute.ThrMma,+ tOtOs: tuple[cute.Tensor],+ sO: cute.Tensor,+ mbar_ptr,+ mbar_softmax_corr_full_offset: Int32,+ mbar_softmax_corr_empty_offset: Int32,+ mbar_P_full_O_rescaled_offset: Int32,+ mbar_P_full_2_offset: Int32,+ mbar_corr_epi_full_offset: Int32,+ mbar_corr_epi_empty_offset: Int32,+ softmax_corr_consumer_phase: Int32,+ o_corr_consumer_phase: Int32,+ corr_epi_producer_phase: Int32,+ softmax_scale_log2: Float32,+ mO_cur: Optional[cute.Tensor] = None,+ gO: Optional[cute.Tensor] = None,+ gmem_tiled_copy_O: Optional[cute.TiledCopy] = None,+):+ """Handle the block-sparse case where a tile is fully masked:+ * zero staged results+ * seed stats+ * satisfy the usual barrier protocol so downstream warps continue to make progress.+ """+ LOG2_E = Float32(math.log2(math.e))++ for stage in cutlass.range_constexpr(q_stage):+ row_sum_value = Float32(1.0)+ row_max_value = (+ -Float32.inf if const_expr(mLSE is not None or learnable_sink is not None) else None+ )+ if const_expr(learnable_sink is not None):+ sink_val = -Float32.inf+ if const_expr(not pack_gqa):+ sink_val = Float32(learnable_sink[head_idx])+ elif tidx < m_block_size:+ q_head_idx = (+ (q_stage * m_block + stage) * m_block_size + tidx+ ) % qhead_per_kvhead + head_idx * qhead_per_kvhead+ sink_val = Float32(learnable_sink[q_head_idx])+ if sink_val != -Float32.inf and (const_expr(not is_split_kv) or split_idx == 0):+ if row_max_value == -Float32.inf:+ row_max_value = sink_val * (LOG2_E / softmax_scale_log2)+ row_sum_value = Float32(1.0)+ else:+ row_sum_value = row_sum_value + utils.exp2f(+ sink_val * LOG2_E - row_max_value * softmax_scale_log2+ )+ if tidx < m_block_size:+ scale_row_idx = tidx + stage * m_block_size+ sScale[scale_row_idx] = row_sum_value+ if const_expr(mLSE is not None or learnable_sink is not None):+ sScale[scale_row_idx + m_block_size * 2] = row_max_value+ acc_flag = row_sum_value == Float32(0.0) or row_sum_value != row_sum_value+ stats[stage] = (row_sum_value, row_max_value, acc_flag)++ cute.arch.mbarrier_wait(+ mbar_ptr + mbar_softmax_corr_full_offset + stage,+ softmax_corr_consumer_phase,+ )+ cute.arch.mbarrier_arrive(mbar_ptr + mbar_softmax_corr_empty_offset + stage)++ if const_expr(gmem_tiled_copy_O is None):+ cute.arch.mbarrier_wait(+ mbar_ptr + mbar_corr_epi_empty_offset + stage,+ corr_epi_producer_phase,+ )+ correction_epilogue(+ thr_mma_pv,+ tOtOs[stage],+ tidx,+ stage,+ m_block,+ seqlen.seqlen_q,+ Float32(0.0), # zero scale ensures empty tile writes zeros into staged outputs+ sO[None, None, stage],+ mO_cur,+ gO,+ gmem_tiled_copy_O,+ )+ if const_expr(gmem_tiled_copy_O is None):+ cute.arch.mbarrier_arrive(mbar_ptr + mbar_corr_epi_full_offset + stage)+ cute.arch.mbarrier_arrive(mbar_ptr + mbar_P_full_O_rescaled_offset + stage)+ cute.arch.mbarrier_arrive(mbar_ptr + mbar_P_full_2_offset + stage)++ softmax_corr_consumer_phase ^= 1+ o_corr_consumer_phase ^= 1+ corr_epi_producer_phase ^= 1++ return (+ softmax_corr_consumer_phase,+ o_corr_consumer_phase,+ corr_epi_producer_phase,+ )+++def softmax_block_sparse_sm100(+ blocksparse_tensors: BlockSparseTensors,+ batch_idx,+ head_idx,+ m_block,+ softmax_step: Callable,+ mask_fn: Callable,+ mask_fn_none: Callable,+ mma_si_consumer_phase: Int32,+ si_corr_producer_phase: Int32,+ s0_s1_sequence_phase: Int32,+ mbar_ptr,+ mbar_softmax_corr_full_offset: Int32,+ mbar_softmax_corr_empty_offset: Int32,+ mbar_P_full_O_rescaled_offset: Int32,+ mbar_P_full_2_offset: Int32,+ q_stage: cutlass.Constexpr,+ stage_idx: Int32,+):+ mask_block_cnt, mask_block_idx, full_block_cnt, full_block_idx = blocksparse_tensors++ curr_mask_block_cnt = mask_block_cnt[batch_idx, head_idx, m_block]+ curr_mask_block_idx = mask_block_idx[batch_idx, head_idx, m_block, None]++ if const_expr(full_block_cnt is not None):+ curr_full_block_cnt = full_block_cnt[batch_idx, head_idx, m_block]+ curr_full_block_idx = full_block_idx[batch_idx, head_idx, m_block, None]+ else:+ curr_full_block_cnt = Int32(0)+ curr_full_block_idx = None++ total_block_cnt = curr_mask_block_cnt + curr_full_block_cnt++ if total_block_cnt == 0:+ cute.arch.mbarrier_arrive(mbar_ptr + mbar_softmax_corr_full_offset + stage_idx)+ cute.arch.mbarrier_arrive(mbar_ptr + mbar_P_full_O_rescaled_offset + stage_idx)+ cute.arch.mbarrier_arrive(mbar_ptr + mbar_P_full_2_offset + stage_idx)+ cute.arch.mbarrier_arrive(mbar_ptr + mbar_softmax_corr_empty_offset + stage_idx)+ else:+ if curr_mask_block_cnt > 0:+ mask_n_block = curr_mask_block_idx[curr_mask_block_cnt - 1]+ (+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ ) = softmax_step(+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ mask_n_block,+ is_first=True,+ mask_fn=partial(mask_fn, mask_seqlen=True), # last block could oob+ )+ for i in cutlass.range(1, curr_mask_block_cnt):+ mask_n_block = curr_mask_block_idx[curr_mask_block_cnt - 1 - i]+ (+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ ) = softmax_step(+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ mask_n_block,+ mask_fn=partial(mask_fn, mask_seqlen=False),+ )++ if curr_full_block_cnt > 0:+ full_n_block = curr_full_block_idx[curr_full_block_cnt - 1]+ if curr_mask_block_cnt == 0:+ (+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ ) = softmax_step(+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ full_n_block,+ is_first=True,+ mask_fn=partial(mask_fn_none, mask_seqlen=True),+ )+ else:+ (+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ ) = softmax_step(+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ full_n_block,+ is_first=False,+ mask_fn=partial(mask_fn_none, mask_seqlen=False),+ )+ for i in cutlass.range(1, curr_full_block_cnt):+ full_n_block = curr_full_block_idx[curr_full_block_cnt - 1 - i]+ (+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ ) = softmax_step(+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ full_n_block,+ mask_fn=partial(mask_fn_none, mask_seqlen=False),+ )++ return (+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ total_block_cnt == 0,+ )diff --git a/src/nvfp4_decode_kernel/_fa4/block_sparsity.py b/src/nvfp4_decode_kernel/_fa4/block_sparsity.pynew file mode 100644index 0000000..48cd3a9--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/block_sparsity.py@@ -0,0 +1,674 @@+"""+Computes block-sparse attention masks for Flex Attention.++This utility generates block sparsity patterns based on common attention masking+strategies (e.g., causal, sliding window). The resulting tensors define which+blocks are fully computed, which are partially computed (requiring a mask), and+which are skipped entirely. This is a temporary solution intended to be replaced+by a more robust preprocessing kernel in the future.+"""++from typing import Tuple, Optional, Callable, List, NamedTuple+import torch+import cutlass.cute as cute+from cutlass.cute.runtime import from_dlpack+++def ceildiv(a: int, b: int) -> int:+ return (a + b - 1) // b+++# placeholder+Config = type("Config", (), {})+++class BlockSparseTensors(NamedTuple):+ mask_block_cnt: cute.Tensor+ mask_block_idx: cute.Tensor+ full_block_cnt: Optional[cute.Tensor]+ full_block_idx: Optional[cute.Tensor]++ def __new_from_mlir_values__(self, values):+ if len(values) == 2:+ values = (*values, None, None)+ return BlockSparseTensors(*values)+++class BlockSparseTensorsTorch(NamedTuple):+ mask_block_cnt: torch.Tensor+ mask_block_idx: torch.Tensor+ full_block_cnt: Optional[torch.Tensor] = None+ full_block_idx: Optional[torch.Tensor] = None+++def _expand_sparsity_tensor(+ tensor: torch.Tensor,+ expected_shape: Tuple[int, ...],+ tensor_name: str,+) -> torch.Tensor:+ """Check if we need to expand the tensor to expected shape, and do so if possible."""+ needs_expand = tensor.shape != expected_shape+ if not needs_expand:+ return tensor+ can_expand = all(map(lambda cur, tgt: cur == tgt or cur == 1, tensor.shape, expected_shape))+ if not can_expand:+ raise ValueError(+ f"{tensor_name} with shape {tensor.shape} cannot be expanded to expected shape {expected_shape}."+ )+ return tensor.expand(*expected_shape).contiguous()+++def _check_and_expand_block(+ name: str,+ cnt: Optional[torch.Tensor],+ idx: Optional[torch.Tensor],+ expected_count_shape: Tuple[int, int, int],+ expected_index_shape: Tuple[int, int, int, int],+) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:+ if (cnt is None) != (idx is None):+ raise ValueError(+ f"{name}_block_cnt and {name}_block_idx must both be provided or both be None"+ )+ if cnt is None or idx is None:+ return None, None+ if cnt.dtype != torch.int32 or idx.dtype != torch.int32:+ raise ValueError(f"{name}_block tensors must have dtype torch.int32")+ if cnt.device != idx.device:+ raise ValueError(f"{name}_block_cnt and {name}_block_idx must be on the same device")+ if not cnt.is_cuda or not idx.is_cuda:+ raise ValueError(f"{name}_block tensors must live on CUDA")+ expanded_cnt = _expand_sparsity_tensor(cnt, expected_count_shape, f"{name}_block_cnt")+ expanded_idx = _expand_sparsity_tensor(idx, expected_index_shape, f"{name}_block_idx")+ return expanded_cnt, expanded_idx+++def get_block_sparse_expected_shapes(+ batch_size: int,+ num_head: int,+ seqlen_q: int,+ seqlen_k: int,+ m_block_size: int,+ n_block_size: int,+ compute_capability: int,+) -> Tuple[Tuple[int, int, int], Tuple[int, int, int, int]]:+ """Return (expected_count_shape, expected_index_shape) for block sparse normalization."""+ # TODO: This multiplier should really be q_stage, wire up in later PR+ # 1 cta handles 2*tile_m rows on SM100+ m_block_size_effective = 2 * m_block_size if compute_capability == 10 else m_block_size+ expected_m_blocks = ceildiv(seqlen_q, m_block_size_effective)+ expected_n_blocks = ceildiv(seqlen_k, n_block_size)+ expected_count_shape = (batch_size, num_head, expected_m_blocks)+ expected_index_shape = (batch_size, num_head, expected_m_blocks, expected_n_blocks)+ return expected_count_shape, expected_index_shape+++def normalize_block_sparse_tensors(+ tensors: BlockSparseTensorsTorch,+ *,+ expected_count_shape: Tuple[int, int, int],+ expected_index_shape: Tuple[int, int, int, int],+) -> BlockSparseTensorsTorch:+ if tensors.mask_block_cnt is None or tensors.mask_block_idx is None:+ raise ValueError("mask_block_cnt and mask_block_idx must be provided for block sparsity.")++ mask_cnt, mask_idx = _check_and_expand_block(+ "mask",+ tensors.mask_block_cnt,+ tensors.mask_block_idx,+ expected_count_shape,+ expected_index_shape,+ )+ if mask_cnt is None or mask_idx is None:+ raise ValueError("mask_block_cnt and mask_block_idx must be provided for block sparsity.")++ full_cnt, full_idx = _check_and_expand_block(+ "full",+ tensors.full_block_cnt,+ tensors.full_block_idx,+ expected_count_shape,+ expected_index_shape,+ )+ if full_cnt is not None and mask_cnt.device != full_cnt.device:+ raise ValueError("All block sparse tensors must be on the same device")++ return BlockSparseTensorsTorch(+ mask_block_cnt=mask_cnt,+ mask_block_idx=mask_idx,+ full_block_cnt=full_cnt,+ full_block_idx=full_idx,+ )+++def is_block_sparsity_enabled(tensors: BlockSparseTensorsTorch) -> bool:+ return any(t is not None for t in (tensors.full_block_cnt, tensors.mask_block_cnt))+++def to_cute_block_sparse_tensors(tensors: BlockSparseTensorsTorch) -> Optional[BlockSparseTensors]:+ if not is_block_sparsity_enabled(tensors):+ return None++ mask_block_cnt_tensor = from_dlpack(+ tensors.mask_block_cnt.detach(), assumed_align=4+ ).mark_layout_dynamic(leading_dim=2)+ mask_block_idx_tensor = from_dlpack(+ tensors.mask_block_idx.detach(), assumed_align=4+ ).mark_layout_dynamic(leading_dim=3)+ full_block_cnt_tensor = (+ from_dlpack(tensors.full_block_cnt.detach(), assumed_align=4).mark_layout_dynamic(+ leading_dim=2+ )+ if tensors.full_block_cnt is not None+ else None+ )+ full_block_idx_tensor = (+ from_dlpack(tensors.full_block_idx.detach(), assumed_align=4).mark_layout_dynamic(+ leading_dim=3+ )+ if tensors.full_block_idx is not None+ else None+ )++ return BlockSparseTensors(+ mask_block_cnt_tensor,+ mask_block_idx_tensor,+ full_block_cnt_tensor,+ full_block_idx_tensor,+ )+++def compute_block_sparsity(+ config: Config,+ mask_mod_flex: Optional[Callable],+ device: str,+ cu_seqlens_q: Optional[torch.Tensor] = None,+ cu_seqlens_k: Optional[torch.Tensor] = None,+ aux_tensors: Optional[List[torch.Tensor]] = None,+) -> Tuple[+ Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor]+]:+ """+ Computes block sparsity tensors from a given masking function.++ This function serves as the main entry point for generating block-sparse masks.+ It dispatches to specialized handlers for variable-length and fixed-length+ sequences.++ Args:+ config: A configuration object containing model and tiling parameters.+ mask_mod_flex: The mask function for generic flex attention patterns.+ device: The device to create tensors on (e.g., 'cuda').+ cu_seqlens_q: Cumulative sequence lengths for Q (for varlen).+ cu_seqlens_k: Cumulative sequence lengths for K (for varlen).+ aux_tensors: A list of auxiliary tensors, e.g., for document masking.++ Returns:+ A tuple of four tensors:+ - `full_block_cnt`: (batch, nheads, n_blocks_q) - Count of full n blocks per m block.+ - `full_block_idx`: (batch, nheads, n_blocks_q, max_n_blocks) - Indices of full n blocks.+ - `mask_block_cnt`: (batch, nheads, n_blocks_q) - Count of partial n blocks per m block.+ - `mask_block_idx`: (batch, nheads, n_blocks_q, max_n_blocks) - Indices of partial n blocks.+ Returns (None, None, None, None) if masking is disabled.+ """+ if not config.use_mask_mod or mask_mod_flex is None:+ return None, None, None, None++ if cu_seqlens_q is not None:+ # Handle variable-length sequences+ return _compute_varlen_sparsity(config, mask_mod_flex, device, cu_seqlens_q, cu_seqlens_k)+ else:+ # Handle fixed-length sequences+ return _compute_sparsity(config, device, aux_tensors)+++## ---------------------------------------------------------------------------+## Fixed-Length Sequence Kernels+## ---------------------------------------------------------------------------+++def _compute_sparsity(+ config: Config, device: str, aux_tensors: Optional[List[torch.Tensor]]+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:+ """Computes block sparsity for fixed-length sequences."""+ n_blocks_q = ceildiv(config.seqlen_q, config.tile_m)+ n_blocks_k = ceildiv(config.seqlen_k, config.tile_n)++ # Pre-allocate output tensors+ full_block_cnt = torch.zeros(+ (config.batch_size, config.nheads, n_blocks_q), device=device, dtype=torch.int32+ )+ mask_block_cnt = torch.zeros(+ (config.batch_size, config.nheads, n_blocks_q), device=device, dtype=torch.int32+ )+ full_block_idx = torch.zeros(+ (config.batch_size, config.nheads, n_blocks_q, n_blocks_k), device=device, dtype=torch.int32+ )+ mask_block_idx = torch.zeros(+ (config.batch_size, config.nheads, n_blocks_q, n_blocks_k), device=device, dtype=torch.int32+ )++ # --- Identity Mask ---+ # All blocks are fully computed.+ if config.mask_mod_name == "identity":+ k_blocks = torch.arange(n_blocks_k, device=device)+ for q_block_idx in range(n_blocks_q):+ full_block_cnt[:, :, q_block_idx] = n_blocks_k+ full_block_idx[:, :, q_block_idx, :n_blocks_k] = k_blocks++ # --- Identity Partial Mask ---+ # All blocks are partially computed (masked).+ elif config.mask_mod_name == "identity_partial":+ k_blocks = torch.arange(n_blocks_k, device=device)+ for q_block_idx in range(n_blocks_q):+ mask_block_cnt[:, :, q_block_idx] = n_blocks_k+ mask_block_idx[:, :, q_block_idx, :n_blocks_k] = k_blocks++ # --- Block Causal Mask ---+ elif config.mask_mod_name == "block_causal":+ k_blocks = torch.arange(n_blocks_k, device=device)+ for q_block_idx in range(n_blocks_q):+ causal_indices = k_blocks[k_blocks <= q_block_idx]+ num_causal_indices = len(causal_indices)+ if num_causal_indices > 0:+ full_block_cnt[:, :, q_block_idx] = num_causal_indices+ full_block_idx[:, :, q_block_idx, :num_causal_indices] = causal_indices++ # --- Causal and Sliding Window Masks ---+ elif config.mask_mod_name in ["causal", "sliding_window"]:+ q_block_indices = torch.arange(n_blocks_q, device=device)+ k_block_indices = torch.arange(n_blocks_k, device=device)++ q_starts = q_block_indices * config.tile_m+ q_ends = torch.minimum(+ (q_block_indices + 1) * config.tile_m, torch.tensor(config.seqlen_q, device=device)+ )+ k_starts = k_block_indices * config.tile_n+ k_ends = torch.minimum(+ (k_block_indices + 1) * config.tile_n, torch.tensor(config.seqlen_k, device=device)+ )++ # Expand dims for broadcasting: (n_blocks_q, 1) and (1, n_blocks_k)+ q_starts, q_ends = q_starts.unsqueeze(1), q_ends.unsqueeze(1)+ k_starts, k_ends = k_starts.unsqueeze(0), k_ends.unsqueeze(0)++ offset = config.seqlen_k - config.seqlen_q++ if config.mask_mod_name == "causal":+ is_full = (k_ends - 1) <= (q_starts + offset)+ # min(k_pos) <= max(q_pos) AND not is_full.+ is_partial = (k_starts <= (q_ends - 1 + offset)) & ~is_full++ else: # sliding_window+ window_size = getattr(config, "window_size", 1024)+ is_full = (k_ends - 1 <= q_starts + offset) & (+ k_starts >= q_ends - 1 + offset - (window_size - 1)+ )+ # A block is EMPTY if no (q, k) pairs satisfy the constraint.+ is_empty = (k_starts > q_ends - 1 + offset) | (+ k_ends - 1 < q_starts + offset - (window_size - 1)+ )+ # A block is PARTIAL if it's not empty and not full.+ is_partial = ~is_empty & ~is_full++ # Populate indices based on the computed block classifications+ for q_block_idx in range(n_blocks_q):+ full_indices = k_block_indices[is_full[q_block_idx]]+ if len(full_indices) > 0:+ full_block_cnt[:, :, q_block_idx] = len(full_indices)+ full_block_idx[:, :, q_block_idx, : len(full_indices)] = full_indices++ partial_indices = k_block_indices[is_partial[q_block_idx]]+ if len(partial_indices) > 0:+ mask_block_cnt[:, :, q_block_idx] = len(partial_indices)+ mask_block_idx[:, :, q_block_idx, : len(partial_indices)] = partial_indices++ elif config.mask_mod_name == "document":+ raise NotImplementedError("Block sparsity for document masking not yet implemented")++ return full_block_cnt, full_block_idx, mask_block_cnt, mask_block_idx+++## ---------------------------------------------------------------------------+## Variable-Length Sequence Kernels+## ---------------------------------------------------------------------------+++def _compute_varlen_sparsity(+ config: Config,+ mask_mod_flex: Callable,+ device: str,+ cu_seqlens_q: torch.Tensor,+ cu_seqlens_k: torch.Tensor,+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:+ """Computes block sparsity for variable-length sequences."""+ assert cu_seqlens_k is not None, "cu_seqlens_k is required for varlen attention"+ assert cu_seqlens_q.shape[0] == config.batch_size + 1+ assert cu_seqlens_k.shape[0] == config.batch_size + 1++ # In varlen, each sequence can have a different number of Q blocks.+ # We pad up to the maximum number of Q blocks in the batch.+ max_m_blocks = 0+ for seq_idx in range(config.batch_size):+ seq_len_q = (cu_seqlens_q[seq_idx + 1] - cu_seqlens_q[seq_idx]).item()+ n_blocks_q = ceildiv(seq_len_q, config.tile_m)+ max_m_blocks = max(max_m_blocks, n_blocks_q)++ # The number of K blocks is determined by the total length of all sequences.+ total_k_len = cu_seqlens_k[-1].item()+ max_n_blocks = ceildiv(total_k_len, config.tile_n)++ # Pre-allocate padded output tensors+ full_block_cnt = torch.zeros(+ (config.batch_size, config.nheads, max_m_blocks), device=device, dtype=torch.int32+ )+ mask_block_cnt = torch.zeros(+ (config.batch_size, config.nheads, max_m_blocks), device=device, dtype=torch.int32+ )+ full_block_idx = torch.zeros(+ (config.batch_size, config.nheads, max_m_blocks, max_n_blocks),+ device=device,+ dtype=torch.int32,+ )+ mask_block_idx = torch.zeros(+ (config.batch_size, config.nheads, max_m_blocks, max_n_blocks),+ device=device,+ dtype=torch.int32,+ )++ # Process each sequence in the batch individually+ for seq_idx in range(config.batch_size):+ seq_start_q = cu_seqlens_q[seq_idx].item()+ seq_end_q = cu_seqlens_q[seq_idx + 1].item()+ seq_len_q = seq_end_q - seq_start_q++ seq_start_k = cu_seqlens_k[seq_idx].item()+ seq_end_k = cu_seqlens_k[seq_idx + 1].item()+ seq_len_k = seq_end_k - seq_start_k++ n_blocks_q = ceildiv(seq_len_q, config.tile_m)+ n_blocks_k = ceildiv(seq_len_k, config.tile_n)++ # Global block indices are relative to the start of the entire batch tensor+ first_m_block_global = seq_start_q // config.tile_m+ first_n_block_global = seq_start_k // config.tile_n++ common_args = {+ "full_block_cnt": full_block_cnt,+ "full_block_idx": full_block_idx,+ "mask_block_cnt": mask_block_cnt,+ "mask_block_idx": mask_block_idx,+ "seq_idx": seq_idx,+ "n_blocks_q": n_blocks_q,+ "n_blocks_k": n_blocks_k,+ "seq_start_q": seq_start_q,+ "seq_end_q": seq_end_q,+ "seq_start_k": seq_start_k,+ "seq_end_k": seq_end_k,+ "first_n_block_global": first_n_block_global,+ "tile_m": config.tile_m,+ "tile_n": config.tile_n,+ "device": device,+ }++ if config.mask_mod_name == "causal":+ _compute_causal_varlen_blocks(**common_args)+ elif config.mask_mod_name == "sliding_window":+ window_size = getattr(config, "window_size", 1024)+ _compute_sliding_window_varlen_blocks(**common_args, window_size=window_size)+ elif config.mask_mod_name == "identity":+ _compute_identity_varlen_blocks(+ full_block_cnt,+ full_block_idx,+ seq_idx,+ n_blocks_q,+ n_blocks_k,+ first_n_block_global,+ device,+ )+ else:+ # Generic case relies on sampling the user-provided mask function+ _compute_generic_varlen_blocks(+ **common_args,+ mask_mod_flex=mask_mod_flex,+ seq_len_q=seq_len_q,+ seq_len_k=seq_len_k,+ num_heads=config.nheads,+ nheads_kv=config.nheads_kv,+ )++ return full_block_cnt, full_block_idx, mask_block_cnt, mask_block_idx+++def _classify_varlen_block(+ m_local: int,+ n_local: int,+ seq_start_q: int,+ seq_end_q: int,+ seq_start_k: int,+ seq_end_k: int,+ tile_m: int,+ tile_n: int,+ is_full_fn: Callable,+ is_partial_fn: Callable,+) -> Tuple[bool, bool]:+ """Helper to classify a varlen block as full, partial, or empty."""+ m_start_global = seq_start_q + m_local * tile_m+ m_end_global = min(seq_start_q + (m_local + 1) * tile_m, seq_end_q)+ n_start_global = seq_start_k + n_local * tile_n+ n_end_global = min(seq_start_k + (n_local + 1) * tile_n, seq_end_k)++ # Use sequence-local coordinates for the logical check+ m_start_local = m_start_global - seq_start_q+ m_end_local = m_end_global - seq_start_q+ n_start_local = n_start_global - seq_start_k+ n_end_local = n_end_global - seq_start_k++ is_full = is_full_fn(m_start_local, m_end_local, n_start_local, n_end_local)+ is_partial = (+ is_partial_fn(m_start_local, m_end_local, n_start_local, n_end_local) and not is_full+ )++ # Any block that touches the sequence boundary is partial because it requires masking.+ at_boundary = (m_end_global > seq_end_q) or (n_end_global > seq_end_k)++ return is_full and not at_boundary, is_partial or (is_full and at_boundary)+++def _compute_causal_varlen_blocks(+ full_block_cnt,+ full_block_idx,+ mask_block_cnt,+ mask_block_idx,+ seq_idx,+ n_blocks_q,+ n_blocks_k,+ seq_start_q,+ seq_end_q,+ seq_start_k,+ seq_end_k,+ first_n_block_global,+ tile_m,+ tile_n,+ device,+ **kwargs,+):+ """Computes causal block sparsity for a single varlen sequence."""+ is_full_fn = lambda m_start, m_end, n_start, n_end: (m_start >= n_end - 1)+ is_partial_fn = lambda m_start, m_end, n_start, n_end: (m_end - 1 >= n_start)++ for m_local in range(n_blocks_q):+ full_blocks, partial_blocks = [], []+ for n_local in range(n_blocks_k):+ is_full, is_partial = _classify_varlen_block(+ m_local,+ n_local,+ seq_start_q,+ seq_end_q,+ seq_start_k,+ seq_end_k,+ tile_m,+ tile_n,+ is_full_fn,+ is_partial_fn,+ )+ n_block_global = first_n_block_global + n_local+ if is_full:+ full_blocks.append(n_block_global)+ elif is_partial:+ partial_blocks.append(n_block_global)++ if full_blocks:+ full_block_cnt[seq_idx, :, m_local] = len(full_blocks)+ full_block_idx[seq_idx, :, m_local, : len(full_blocks)] = torch.tensor(+ full_blocks, device=device+ )+ if partial_blocks:+ mask_block_cnt[seq_idx, :, m_local] = len(partial_blocks)+ mask_block_idx[seq_idx, :, m_local, : len(partial_blocks)] = torch.tensor(+ partial_blocks, device=device+ )+++def _compute_sliding_window_varlen_blocks(+ full_block_cnt,+ full_block_idx,+ mask_block_cnt,+ mask_block_idx,+ seq_idx,+ n_blocks_q,+ n_blocks_k,+ seq_start_q,+ seq_end_q,+ seq_start_k,+ seq_end_k,+ first_n_block_global,+ tile_m,+ tile_n,+ window_size,+ device,+ **kwargs,+):+ """Computes sliding window block sparsity for a single varlen sequence."""+ is_full_fn = lambda m_start, m_end, n_start, n_end: (n_end - 1 <= m_start) and (+ n_start >= m_start - window_size + 1+ )+ is_partial_fn = lambda m_start, m_end, n_start, n_end: not (+ (n_start > m_end - 1) or (n_end - 1 < m_start - window_size + 1)+ )++ for m_local in range(n_blocks_q):+ full_blocks, partial_blocks = [], []+ for n_local in range(n_blocks_k):+ is_full, is_partial = _classify_varlen_block(+ m_local,+ n_local,+ seq_start_q,+ seq_end_q,+ seq_start_k,+ seq_end_k,+ tile_m,+ tile_n,+ is_full_fn,+ is_partial_fn,+ )+ n_block_global = first_n_block_global + n_local+ if is_full:+ full_blocks.append(n_block_global)+ elif is_partial:+ partial_blocks.append(n_block_global)++ if full_blocks:+ full_block_cnt[seq_idx, :, m_local] = len(full_blocks)+ full_block_idx[seq_idx, :, m_local, : len(full_blocks)] = torch.tensor(+ full_blocks, device=device+ )+ if partial_blocks:+ mask_block_cnt[seq_idx, :, m_local] = len(partial_blocks)+ mask_block_idx[seq_idx, :, m_local, : len(partial_blocks)] = torch.tensor(+ partial_blocks, device=device+ )+++def _compute_identity_varlen_blocks(+ full_block_cnt,+ full_block_idx,+ seq_idx,+ n_blocks_q,+ n_blocks_k,+ first_n_block_global,+ device,+ **kwargs,+):+ """Computes identity (all-attend) block sparsity for a single varlen sequence."""+ n_blocks_global = torch.arange(+ first_n_block_global, first_n_block_global + n_blocks_k, device=device, dtype=torch.int32+ )+ for m_local in range(n_blocks_q):+ full_block_cnt[seq_idx, :, m_local] = n_blocks_k+ full_block_idx[seq_idx, :, m_local, :n_blocks_k] = n_blocks_global+++def _compute_generic_varlen_blocks(+ full_block_cnt,+ full_block_idx,+ mask_block_cnt,+ mask_block_idx,+ mask_mod_flex,+ seq_idx,+ num_heads,+ n_blocks_q,+ n_blocks_k,+ seq_len_q,+ seq_len_k,+ first_n_block_global,+ tile_m,+ tile_n,+ nheads_kv,+ device,+ **kwargs,+):+ """Generic sampling-based block classification for a varlen sequence."""+ qhead_per_kvhead = num_heads // nheads_kv++ for h_q in range(num_heads):+ h_kv = h_q // qhead_per_kvhead+ for m_local in range(n_blocks_q):+ m_start_local = m_local * tile_m+ m_end_local = min((m_local + 1) * tile_m, seq_len_q)++ full_blocks, partial_blocks = [], []+ for n_local in range(n_blocks_k):+ n_start_local = n_local * tile_n+ n_end_local = min((n_local + 1) * tile_n, seq_len_k)++ # Sample points within the block (corners and center) to classify it.+ # Coordinates are sequence-local, as required by mask_mod_flex.+ sample_positions = [+ (m_start_local, n_start_local),+ (m_start_local, n_end_local - 1),+ (m_end_local - 1, n_start_local),+ (m_end_local - 1, n_end_local - 1),+ ((m_start_local + m_end_local) // 2, (n_start_local + n_end_local) // 2),+ ]++ unmasked_count = sum(+ 1+ for q_pos, k_pos in sample_positions+ if mask_mod_flex(seq_idx, h_q, q_pos, k_pos, seq_len_q, seq_len_k)+ )++ n_block_global = first_n_block_global + n_local+ if unmasked_count == len(sample_positions): # All samples unmasked -> full+ full_blocks.append(n_block_global)+ elif unmasked_count > 0: # Some unmasked -> partial+ partial_blocks.append(n_block_global)++ if full_blocks:+ full_block_cnt[seq_idx, h_q, m_local] = len(full_blocks)+ full_block_idx[seq_idx, h_q, m_local, : len(full_blocks)] = torch.tensor(+ full_blocks, device=device+ )+ if partial_blocks:+ mask_block_cnt[seq_idx, h_q, m_local] = len(partial_blocks)+ mask_block_idx[seq_idx, h_q, m_local, : len(partial_blocks)] = torch.tensor(+ partial_blocks, device=device+ )diff --git a/src/nvfp4_decode_kernel/_fa4/copy_utils.py b/src/nvfp4_decode_kernel/_fa4/copy_utils.pynew file mode 100644index 0000000..07d4bd7--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/copy_utils.py@@ -0,0 +1,341 @@+# Copyright (c) 2025, Wentao Guo, Ted Zadouri, Tri Dao.++import math+from typing import Optional, Type, Tuple, Callable++import cutlass+import cutlass.cute as cute+from cutlass import Float32, Int32, Boolean, const_expr+from cutlass.cute.nvgpu import cpasync+import cutlass.utils.blackwell_helpers as sm100_utils+from cutlass.cutlass_dsl import T, dsl_user_op+from cutlass._mlir.dialects import llvm+import cutlass.pipeline+++@dsl_user_op+def cvt_copy(+ atom: cute.CopyAtom,+ src: cute.Tensor,+ dst: cute.Tensor,+ *,+ pred: Optional[cute.Tensor] = None,+ loc=None,+ ip=None,+ **kwargs,+) -> None:+ assert isinstance(src.iterator, cute.Pointer) and src.memspace == cute.AddressSpace.rmem+ if const_expr(src.element_type != dst.element_type):+ src_cvt = cute.make_fragment_like(src, dst.element_type, loc=loc, ip=ip)+ src_cvt.store(src.load().to(dst.element_type))+ src = src_cvt+ cute.copy(atom, src, dst, pred=pred, loc=loc, ip=ip, **kwargs)+++@dsl_user_op+def load_s2r(src: cute.Tensor, *, loc=None, ip=None) -> cute.Tensor:+ dst = cute.make_fragment_like(src, src.element_type, loc=loc, ip=ip)+ cute.autovec_copy(src, dst, loc=loc, ip=ip)+ return dst+++@dsl_user_op+def get_copy_atom(+ dtype: Type[cutlass.Numeric], num_copy_elems: int, is_async: bool = False, *, loc=None, ip=None+) -> cute.CopyAtom:+ num_copy_bits = const_expr(min(128, num_copy_elems * dtype.width))+ copy_op = cpasync.CopyG2SOp() if is_async else cute.nvgpu.CopyUniversalOp()+ return cute.make_copy_atom(copy_op, dtype, num_bits_per_copy=num_copy_bits)+++@dsl_user_op+def make_tmem_copy(+ tmem_copy_atom: cute.CopyAtom, num_wg: int = 1, *, loc=None, ip=None+) -> cute.CopyAtom:+ num_dp, num_bits, num_rep, _ = sm100_utils.get_tmem_copy_properties(tmem_copy_atom)+ assert num_dp == 32+ assert num_bits == 32+ tiler_mn = (cute.make_layout((128 * num_rep * num_wg // 32, 32), stride=(32, 1)),)+ layout_tv = cute.make_layout(+ ((32, 4, num_wg), (num_rep, 32)), stride=((0, 1, 4 * num_rep), (4, 4 * num_rep * num_wg))+ )+ return cute.make_tiled_copy(tmem_copy_atom, layout_tv, tiler_mn)+++@dsl_user_op+def copy(+ src: cute.Tensor,+ dst: cute.Tensor,+ *,+ pred: Optional[cute.Tensor] = None,+ num_copy_elems: int = 1,+ is_async: bool = False,+ loc=None,+ ip=None,+ **kwargs,+) -> None:+ copy_atom = get_copy_atom(src.element_type, num_copy_elems, is_async)+ cute.copy(copy_atom, src, dst, pred=pred, loc=loc, ip=ip, **kwargs)+++def tiled_copy_1d(+ dtype: Type[cutlass.Numeric], num_threads: int, num_copy_elems: int = 1, is_async: bool = False+) -> cute.TiledCopy:+ num_copy_bits = num_copy_elems * dtype.width+ copy_op = cpasync.CopyG2SOp() if is_async else cute.nvgpu.CopyUniversalOp()+ copy_atom = cute.make_copy_atom(copy_op, dtype, num_bits_per_copy=num_copy_bits)+ thr_layout = cute.make_layout(num_threads)+ val_layout = cute.make_layout(num_copy_elems)+ return cute.make_tiled_copy_tv(copy_atom, thr_layout, val_layout)+++def tiled_copy_2d(+ dtype: Type[cutlass.Numeric], major_mode_size: int, num_threads: int, is_async: bool = False+) -> cute.TiledCopy:+ num_copy_bits = math.gcd(major_mode_size, 128 // dtype.width) * dtype.width+ copy_elems = num_copy_bits // dtype.width+ copy_op = cpasync.CopyG2SOp() if is_async else cute.nvgpu.CopyUniversalOp()+ copy_atom = cute.make_copy_atom(copy_op, dtype, num_bits_per_copy=num_copy_bits)+ gmem_threads_per_row = major_mode_size // copy_elems+ assert num_threads % gmem_threads_per_row == 0+ thr_layout = cute.make_ordered_layout(+ (num_threads // gmem_threads_per_row, gmem_threads_per_row),+ order=(1, 0),+ )+ val_layout = cute.make_layout((1, copy_elems))+ return cute.make_tiled_copy_tv(copy_atom, thr_layout, val_layout)+++@dsl_user_op+def atomic_add_fp32x4(+ a: Float32, b: Float32, c: Float32, d: Float32, gmem_ptr: cute.Pointer, *, loc=None, ip=None+) -> None:+ gmem_ptr_i64 = gmem_ptr.toint(loc=loc, ip=ip).ir_value()+ # cache_hint = cutlass.Int64(0x12F0000000000000)+ llvm.inline_asm(+ None,+ [+ gmem_ptr_i64,+ Float32(a).ir_value(loc=loc, ip=ip),+ Float32(b).ir_value(loc=loc, ip=ip),+ Float32(c).ir_value(loc=loc, ip=ip),+ Float32(d).ir_value(loc=loc, ip=ip),+ ],+ # [gmem_ptr_i64, Float32(a).ir_value(loc=loc, ip=ip), cache_hint.ir_value()],+ "{\n\t"+ # ".reg .b128 abcd;\n\t"+ # "mov.b128 abcd, {$1, $2, $3, $4};\n\t"+ ".reg .v4 .f32 abcd;\n\t"+ # "mov.b128 abcd, {$1, $2, $3, $4};\n\t"+ "mov.f32 abcd.x, $1;\n\t"+ "mov.f32 abcd.y, $2;\n\t"+ "mov.f32 abcd.z, $3;\n\t"+ "mov.f32 abcd.w, $4;\n\t"+ "red.global.add.v4.f32 [$0], abcd;\n\t"+ # "red.global.add.L2::cache_hint.v4.f32 [$0], abcd, 0x14F0000000000000;\n\t"+ "}\n",+ # "red.global.add.L2::cache_hint.f32 [$0], $1, 0x12F0000000000000;",+ # "red.global.add.L2::cache_hint.f32 [$0], $1, $2;",+ "l,f,f,f,f",+ # "l,f,l",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+++@dsl_user_op+def set_block_rank(+ smem_ptr: cute.Pointer, peer_cta_rank_in_cluster: Int32, *, loc=None, ip=None+) -> Int32:+ """Map the given smem pointer to the address at another CTA rank in the cluster."""+ smem_ptr_i32 = smem_ptr.toint(loc=loc, ip=ip).ir_value()+ return Int32(+ llvm.inline_asm(+ T.i32(),+ [smem_ptr_i32, peer_cta_rank_in_cluster.ir_value()],+ "mapa.shared::cluster.u32 $0, $1, $2;",+ "=r,r,r",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ )+++@dsl_user_op+def store_shared_remote_fp32x4(+ a: Float32,+ b: Float32,+ c: Float32,+ d: Float32,+ smem_ptr: cute.Pointer,+ mbar_ptr: cute.Pointer,+ peer_cta_rank_in_cluster: Int32,+ *,+ loc=None,+ ip=None,+) -> None:+ remote_smem_ptr_i32 = set_block_rank(+ smem_ptr, peer_cta_rank_in_cluster, loc=loc, ip=ip+ ).ir_value()+ remote_mbar_ptr_i32 = set_block_rank(+ mbar_ptr, peer_cta_rank_in_cluster, loc=loc, ip=ip+ ).ir_value()+ llvm.inline_asm(+ None,+ [+ remote_smem_ptr_i32,+ remote_mbar_ptr_i32,+ Float32(a).ir_value(loc=loc, ip=ip),+ Float32(b).ir_value(loc=loc, ip=ip),+ Float32(c).ir_value(loc=loc, ip=ip),+ Float32(d).ir_value(loc=loc, ip=ip),+ ],+ "{\n\t"+ ".reg .v4 .f32 abcd;\n\t"+ "mov.f32 abcd.x, $2;\n\t"+ "mov.f32 abcd.y, $3;\n\t"+ "mov.f32 abcd.z, $4;\n\t"+ "mov.f32 abcd.w, $5;\n\t"+ "st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.f32 [$0], abcd, [$1];\n\t"+ "}\n",+ "r,r,f,f,f,f",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+++@dsl_user_op+def cpasync_bulk_g2s(+ gmem_ptr: cute.Pointer,+ smem_ptr: cute.Pointer,+ tma_bar_ptr: cute.Pointer,+ size: int | Int32,+ *,+ loc=None,+ ip=None,+):+ gmem_ptr_i64 = gmem_ptr.toint(loc=loc, ip=ip).ir_value()+ smem_ptr_i32 = smem_ptr.toint(loc=loc, ip=ip).ir_value()+ mbar_ptr_i32 = tma_bar_ptr.toint(loc=loc, ip=ip).ir_value()+ llvm.inline_asm(+ None,+ [gmem_ptr_i64, smem_ptr_i32, mbar_ptr_i32, Int32(size).ir_value()],+ "cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes [$1], [$0], $3, [$2];",+ "l,r,r,r",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+++@dsl_user_op+def cpasync_reduce_bulk_add_f32(+ smem_ptr: cute.Pointer,+ gmem_ptr: cute.Pointer,+ store_bytes: int | Int32,+ *,+ loc=None,+ ip=None,+):+ smem_ptr_i32 = smem_ptr.toint(loc=loc, ip=ip).ir_value()+ # cache_hint = cutlass.Int64(0x14F0000000000000) # EVICT_LAST+ llvm.inline_asm(+ None,+ [gmem_ptr.llvm_ptr, smem_ptr_i32, Int32(store_bytes).ir_value()],+ "cp.reduce.async.bulk.global.shared::cta.bulk_group.add.f32 [$0], [$1], $2;",+ "l,r,r",+ # [gmem_ptr.llvm_ptr, smem_ptr_i32, Int32(store_bytes).ir_value(), cache_hint.ir_value()],+ # "cp.reduce.async.bulk.global.shared::cta.bulk_group.L2::cache_hint.add.f32 [$0], [$1], $2, $3;",+ # "l,r,r,l",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+++def cpasync_bulk_get_copy_fn(+ src_tensor: cute.Tensor,+ dst_tensor: cute.Tensor,+ single_stage: bool = False,+ **kwargs,+) -> Callable:+ # src_is_smem = const_expr(+ # isinstance(src_tensor.iterator, cute.Pointer)+ # and src_tensor.memspace == cute.AddressSpace.smem+ # )+ group_rank_src = const_expr(cute.rank(src_tensor) - (1 if not single_stage else 0))+ group_rank_dst = const_expr(cute.rank(dst_tensor) - (1 if not single_stage else 0))+ # ((atom_v, rest_v), STAGE), ((atom_v, rest_v), RestK)+ src = cute.group_modes(src_tensor, 0, group_rank_src)+ dst = cute.group_modes(dst_tensor, 0, group_rank_dst)++ def copy_bulk(src_idx, dst_idx, **new_kwargs):+ size = const_expr(cute.size(src.shape[:-1]) * src.element_type.width // 8)+ cpasync_bulk_g2s(+ src[None, src_idx].iterator,+ dst[None, dst_idx].iterator,+ size=size,+ **new_kwargs,+ **kwargs+ )++ def copy_bulk_single_stage(**new_kwargs):+ size = const_expr(cute.size(src.shape) * src.element_type.width // 8)+ cpasync_bulk_g2s(src.iterator, dst.iterator, size=size, **new_kwargs, **kwargs)++ return copy_bulk if const_expr(not single_stage) else copy_bulk_single_stage+++def tma_get_copy_fn(+ atom: cute.CopyAtom,+ cta_coord: cute.Coord,+ cta_layout: cute.Layout,+ src_tensor: cute.Tensor,+ dst_tensor: cute.Tensor,+ filter_zeros: bool = False,+ single_stage: bool = False,+ **kwargs,+) -> Callable:+ src_is_smem = const_expr(+ isinstance(src_tensor.iterator, cute.Pointer)+ and src_tensor.memspace == cute.AddressSpace.smem+ )+ smem_tensor, gmem_tensor = (src_tensor, dst_tensor) if src_is_smem else (dst_tensor, src_tensor)+ group_rank_smem = const_expr(cute.rank(smem_tensor) - (1 if not single_stage else 0))+ group_rank_gmem = const_expr(cute.rank(gmem_tensor) - (1 if not single_stage else 0))+ # ((atom_v, rest_v), STAGE), ((atom_v, rest_v), RestK)+ s, g = cpasync.tma_partition(+ atom,+ cta_coord,+ cta_layout,+ cute.group_modes(smem_tensor, 0, group_rank_smem),+ cute.group_modes(gmem_tensor, 0, group_rank_gmem),+ )++ if const_expr(filter_zeros):+ s = cute.filter_zeros(s)+ g = cute.filter_zeros(g)+ src, dst = (s, g) if src_is_smem else (g, s)++ def copy_tma(src_idx, dst_idx, **new_kwargs):+ cute.copy(atom, src[None, src_idx], dst[None, dst_idx], **new_kwargs, **kwargs)++ def copy_tma_single_stage(**new_kwargs):+ cute.copy(atom, src, dst, **new_kwargs, **kwargs)++ return (copy_tma if const_expr(not single_stage) else copy_tma_single_stage), s, g+++def tma_producer_copy_fn(copy: Callable, pipeline: cutlass.pipeline.PipelineAsync):+ def copy_fn(src_idx, producer_state: cutlass.pipeline.PipelineState, **new_kwargs):+ copy(+ src_idx=src_idx,+ dst_idx=producer_state.index,+ tma_bar_ptr=pipeline.producer_get_barrier(producer_state),+ **new_kwargs,+ )++ return copy_fndiff --git a/src/nvfp4_decode_kernel/_fa4/cute_dsl_utils.py b/src/nvfp4_decode_kernel/_fa4/cute_dsl_utils.pynew file mode 100644index 0000000..6deeac3--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/cute_dsl_utils.py@@ -0,0 +1,124 @@+# Copyright (c) 2025, Tri Dao.++import os+import pathlib+from typing import Tuple+from functools import partial, lru_cache+from dataclasses import dataclass, fields++import torch++try:+ from triton.tools.disasm import extract+except ImportError:+ extract = None++import cutlass+import cutlass.cute as cute+from cutlass.base_dsl.typing import JitArgument+from cutlass.cutlass_dsl import NumericMeta+++StaticTypes = (cutlass.Constexpr, NumericMeta, int, bool, str, float, type(None))+++load_cubin_module_data_og = cutlass.base_dsl.runtime.cuda.load_cubin_module_data+cute_compile_og = cute.compile+++torch2cute_dtype_map = {+ torch.float16: cutlass.Float16,+ torch.bfloat16: cutlass.BFloat16,+ torch.float32: cutlass.Float32,+}+++@lru_cache+def get_max_active_clusters(cluster_size):+ return cutlass.utils.HardwareInfo().get_max_active_clusters(cluster_size=cluster_size)+++@lru_cache+def get_device_capacity(device: torch.device = None) -> Tuple[int, int]:+ return torch.cuda.get_device_capability(device)+++@dataclass+class ParamsBase:+ def __extract_mlir_values__(self):+ all_fields = [getattr(self, field.name) for field in fields(self)]+ non_constexpr_fields = [f for f in all_fields if not isinstance(f, StaticTypes)]+ values, self._values_pos = [], []+ for obj in non_constexpr_fields:+ obj_values = cutlass.extract_mlir_values(obj)+ values += obj_values+ self._values_pos.append(len(obj_values))+ return values++ def __new_from_mlir_values__(self, values):+ all_fields = {field.name: getattr(self, field.name) for field in fields(self)}+ constexpr_fields = {n: f for n, f in all_fields.items() if isinstance(f, StaticTypes)}+ non_constexpr_fields = {+ n: f for n, f in all_fields.items() if not isinstance(f, StaticTypes)+ }+ for (name, field), n_items in zip(non_constexpr_fields.items(), self._values_pos):+ non_constexpr_fields[name] = cutlass.new_from_mlir_values(field, values[:n_items])+ values = values[n_items:]+ return self.__class__(**non_constexpr_fields, **constexpr_fields)+++@dataclass+class ArgumentsBase(JitArgument):+ def __c_pointers__(self):+ all_fields = [getattr(self, field.name) for field in fields(self)]+ non_constexpr_fields = [f for f in all_fields if not isinstance(f, StaticTypes)]+ c_ptrs = []+ for obj in non_constexpr_fields:+ if hasattr(obj, "__c_pointers__"):+ c_ptrs.extend(obj.__c_pointers__())+ return c_ptrs++ def __get_mlir_types__(self):+ all_fields = [getattr(self, field.name) for field in fields(self)]+ non_constexpr_fields = [f for f in all_fields if not isinstance(f, StaticTypes)]+ types, self._values_pos = [], []+ for obj in non_constexpr_fields:+ if hasattr(obj, "__get_mlir_types__"):+ obj_types = obj.__get_mlir_types__()+ types.extend(obj_types)+ self._values_pos.append(len(obj_types))+ else:+ self._values_pos.append(0)+ return types++ def __new_from_mlir_values__(self, values):+ all_fields = {field.name: getattr(self, field.name) for field in fields(self)}+ constexpr_fields = {n: f for n, f in all_fields.items() if isinstance(f, StaticTypes)}+ non_constexpr_fields = {+ n: f for n, f in all_fields.items() if not isinstance(f, StaticTypes)+ }+ for (name, field), n_items in zip(non_constexpr_fields.items(), self._values_pos):+ non_constexpr_fields[name] = cutlass.new_from_mlir_values(field, values[:n_items])+ values = values[n_items:]+ return self.__class__(**non_constexpr_fields, **constexpr_fields)+++def load_cubin_module_data_patched(cubin_data, filepath):+ pathlib.Path(filepath).write_bytes(cubin_data)+ return load_cubin_module_data_og(cubin_data)+++def cute_compile_patched(*args, **kwargs):+ """A patched version of cute.compile that dump the SASS to a file if CUTE_CUBIN_PATH is set."""+ cubin_path = os.getenv("CUTE_CUBIN_PATH", None)+ if cubin_path is not None:+ cutlass.base_dsl.runtime.cuda.load_cubin_module_data = partial(+ load_cubin_module_data_patched, filepath=cubin_path+ )+ output = cute_compile_og(*args, **kwargs)+ if cubin_path is not None:+ cutlass.base_dsl.runtime.cuda.load_cubin_module_data = load_cubin_module_data_og+ if extract is not None:+ sass = extract(cubin_path, None)+ pathlib.Path(cubin_path).with_suffix(".annotated.sass").write_text(sass)+ return outputdiff --git a/src/nvfp4_decode_kernel/_fa4/fast_math.py b/src/nvfp4_decode_kernel/_fa4/fast_math.pynew file mode 100644index 0000000..943388f--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/fast_math.py@@ -0,0 +1,97 @@+# Copyright (c) 2025, Tri Dao.++from typing import Tuple++import cutlass+import cutlass.cute as cute+from cutlass import Int32, Uint32+from cutlass.cutlass_dsl import T, dsl_user_op+from cutlass._mlir.dialects import llvm+++def clz(x: Int32) -> Int32:+ # for i in cutlass.range_constexpr(32):+ # if (1 << (31 - i)) & x:+ # return Int32(i)+ # return Int32(32)+ # Early exit is not supported yet+ res = Int32(32)+ done = False+ for i in cutlass.range(32):+ if ((1 << (31 - i)) & x) and not done:+ res = Int32(i)+ done = True+ return res+++def find_log2(x: Int32) -> Int32:+ a: Int32 = Int32(31 - clz(x))+ return a + ((x & (x - 1)) != 0) # Round up, add 1 if not a power of 2.+++@dsl_user_op+def umulhi(a: Int32, b: Int32, *, loc=None, ip=None) -> Uint32:+ return Uint32(+ llvm.inline_asm(+ T.i32(),+ [Int32(a).ir_value(loc=loc, ip=ip), Int32(b).ir_value(loc=loc, ip=ip)],+ "mul.hi.u32 $0, $1, $2;",+ "=r,r,r",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ )+++class FastDivmod:+ def __init__(+ self, divisor: Int32, multipler: Uint32, shift_right: Uint32, *, loc=None, ip=None+ ):+ self.divisor = divisor+ self.multiplier = multipler+ self.shift_right = shift_right+ self._loc = loc++ # called by host+ @staticmethod+ def create(divisor: Int32, *, loc=None, ip=None) -> "FastDivmod":+ """Construct the FastDivmod object, in host code.+ This precomputes some values based on the divisor and is computationally expensive.+ """+ p = Uint32(31 + find_log2(divisor))+ divisor_u32 = Uint32(divisor)+ multiplier = Uint32(((cutlass.Uint64(1) << p) + divisor_u32 - 1) // divisor_u32)+ shift_right = Uint32(p - 32)+ return FastDivmod(divisor, multiplier, shift_right, loc=loc, ip=ip)++ @cute.jit+ def div(self, dividend: Int32) -> Int32:+ return (+ Int32(umulhi(dividend, self.multiplier) >> self.shift_right)+ if self.divisor != 1+ else dividend+ )++ def divmod(self, dividend: Int32) -> Tuple[Int32, Int32]:+ quotient = self.div(dividend)+ remainder = dividend - quotient * self.divisor+ return quotient, remainder++ def __extract_mlir_values__(self):+ values, self._values_pos = [], []+ for obj in [self.divisor, self.multiplier, self.shift_right]:+ obj_values = cutlass.extract_mlir_values(obj)+ values += obj_values+ self._values_pos.append(len(obj_values))+ return values++ def __new_from_mlir_values__(self, values):+ obj_list = []+ for obj, n_items in zip(+ [self.divisor, self.multiplier, self.shift_right], self._values_pos+ ):+ obj_list.append(cutlass.new_from_mlir_values(obj, values[:n_items]))+ values = values[n_items:]+ return FastDivmod(*(tuple(obj_list)), loc=self._loc)diff --git a/src/nvfp4_decode_kernel/_fa4/mask.py b/src/nvfp4_decode_kernel/_fa4/mask.pynew file mode 100644index 0000000..afd730d--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/mask.py@@ -0,0 +1,457 @@+# Copyright (c) 2025, Tri Dao.++from typing import Optional, Callable+from dataclasses import dataclass++import cutlass+import cutlass.cute as cute+from cutlass import Float32, Int32, const_expr++from . import utils as utils+++def mask_r2p(X: cute.Tensor, col_limit: Int32, arch: int = 90, rank1: bool = False) -> None:+ # Bit manipulation, compiles down to the R2P instruction+ # For sm100: we know that tScS_t2r[i][1] == i, for the particular tmem copy atom we're using.+ # For sm90: instead of comparing limit to 0, 1, 8, 9, 16, 17, ...,+ # we compare a transformed version of limit to 0, 1, 2, 3, 4, 5, ...+ if const_expr(arch == 90):+ col_limit_transformed = col_limit // 8 * 2 + min(col_limit % 8, 2)+ else:+ col_limit_transformed = col_limit+ ncol = const_expr(cute.size(X.shape[cute.rank(X) - 1]) if not rank1 else cute.size(X.shape))+ # Ideally we'd move by 32 instead of 24, but mask >> i isn't correct for i == 31+ for s in cutlass.range_constexpr(cute.ceil_div(ncol, 24)):+ # Don't need to clamp to 32 since the shr.u32 instruction does that already+ col_limit_right_s = max(col_limit_transformed - s * 24, 0)+ # 0 -> 0b00...00, 1 -> 0b00...01, ..., 31 -> 0b01...11, 32 -> 0b11...11+ mask = (1 << col_limit_right_s) - 1+ # This needs to be range_constexpr, o/w the compiler can't generate the R2P instruction+ for i in cutlass.range_constexpr(min(24, ncol - s * 24)):+ in_bound = cutlass.Boolean(mask & (1 << i))+ c = s * 24 + i+ if const_expr(rank1):+ X[c] = X[c] if in_bound else -Float32.inf+ # This is the equivalent of:+ # X[s * 24 + i] = X[s * 24 + i] if col_limit_right_s <= i else -Float32.inf+ else:+ for r in cutlass.range_constexpr(cute.size(X.shape[0])):+ X[r, c] = X[r, c] if in_bound else -Float32.inf+++def mask_r2p_transposed(X: cute.Tensor, row_limit_top: Int32, num_rep: int) -> None:+ # Bit manipulation, compiles down to the R2P instruction+ # For sm100: we know that tScS_t2r[i][0] has the form 0, 1, ..., 31, 64, ..., 127+ # or 0, 1, ..., 15, 32, ..., 47, 64, ...+ # We compare a transformed version of limit to 0, 1, 2, 3, 4, 5, ...+ # Here we hardcode for the case of 2 warp groups.+ num_wg = 2+ row_limit_top_transformed = row_limit_top // (num_rep * num_wg) * num_rep + min(+ row_limit_top % (num_rep * num_wg), num_rep+ )+ ncol = cute.size(X.shape)+ # Ideally we'd move by 32 instead of 24, but mask >> i isn't correct for i == 31+ for s in cutlass.range_constexpr(cute.ceil_div(ncol, 24)):+ row_limit_top_s = max(row_limit_top_transformed - s * 24, 0)+ # 0 -> 0b00...00, 1 -> 0b00...01, ..., 31 -> 0b01...11, 32 -> 0b11...11+ mask = (1 << row_limit_top_s) - 1+ # This needs to be range_constexpr, o/w the compiler can't generate the R2P instruction+ for i in cutlass.range_constexpr(min(24, ncol - s * 24)):+ out_bound = cutlass.Boolean(mask & (1 << i))+ c = s * 24 + i+ X[c] = -Float32.inf if out_bound else X[c]+ # tidx = cute.arch.thread_idx()[0] % 256+ # if tidx == 128:+ # cute.printf("tidx = {}, s = {}, i = {}, row_limit_top = {}, row_limit_top_s = {}, mask = {}, out_bound = {}", tidx, s, i, row_limit_top, row_limit_top_s, mask, out_bound)+++@dataclass(frozen=True)+class AttentionMask:+ tile_m: cutlass.Constexpr[int]+ tile_n: cutlass.Constexpr[int]+ seqlen_q: Int32+ seqlen_k: Int32+ window_size_left: Optional[Int32] = None+ window_size_right: Optional[Int32] = None+ qhead_per_kvhead_packgqa: cutlass.Constexpr[int] = 1 # only pass in if we're doing PackGQA+ swap_AB: cutlass.Constexpr[bool] = False++ @cute.jit+ def apply_mask(+ self,+ acc_S: cute.Tensor,+ batch_idx: cutlass.Int32,+ head_idx: cutlass.Int32,+ m_block: cutlass.Int32,+ n_block: cutlass.Int32,+ thr_mma: cute.TiledMma,+ mask_seqlen: cutlass.Constexpr[bool],+ mask_causal: cutlass.Constexpr[bool],+ mask_local: cutlass.Constexpr[bool] = False,+ mask_mod: cutlass.Constexpr[Optional[Callable]] = None,+ aux_tensors: Optional[list] = None,+ ) -> None:+ assert not (mask_causal and mask_local), "mask_causal and mask_local cannot be both True"+ acc_S_mn = utils.make_acc_tensor_mn_view(acc_S, transpose=self.swap_AB)+ acc_shape = (self.tile_m, self.tile_n)+ cS = cute.make_identity_tensor(acc_shape if not self.swap_AB else acc_shape[::-1])+ tScS_mn = utils.make_acc_tensor_mn_view(thr_mma.partition_C(cS), transpose=self.swap_AB)+ # We use t0ScS as these indices are known at compile time. We then must subtract the+ # column limit by the thread column offset.+ t0ScS_mn = utils.make_acc_tensor_mn_view(+ thr_mma.get_slice(0).partition_C(cS), transpose=self.swap_AB+ )+ ROW = 0 if const_expr(not self.swap_AB) else 1+ COL = 1 if const_expr(not self.swap_AB) else 0+ thr_col_offset = tScS_mn[0][COL]+ # To handle edge cases of completely masked out rows where n_block_max = 0,+ # we treat negative n_blocks as 0th n_block+ # TODO: find more transparent solution+ if n_block < 0:+ n_block = 0+ seqlenk_col_limit = self.seqlen_k - n_block * self.tile_n - thr_col_offset+ if const_expr(not mask_causal and not mask_local and mask_mod is None):+ if const_expr(mask_seqlen):+ # The compiler now choses not to use R2P+ r2p = const_expr(False and not self.swap_AB)+ if const_expr(not r2p):+ # traverse column index.+ for c in cutlass.range(cute.size(tScS_mn.shape[1]), unroll_full=True):+ oob = t0ScS_mn[0, c][COL] >= seqlenk_col_limit+ for r in cutlass.range(cute.size(tScS_mn.shape[0]), unroll_full=True):+ acc_S_mn[r, c] = -Float32.inf if oob else acc_S_mn[r, c]+ else:+ mask_r2p(acc_S_mn, seqlenk_col_limit, arch=90)++ elif const_expr(+ not mask_causal and not mask_local and mask_mod is not None+ ): # FlexAttention mask mod+ nrow = const_expr(cute.size(tScS_mn.shape[0]))+ ncol = const_expr(cute.size(tScS_mn.shape[1]))+ thr_col_offset = tScS_mn[0, 0][1]++ for r in cutlass.range_constexpr(nrow):+ global_row_idx = tScS_mn[r, 0][0] + m_block * self.tile_m++ for col in cutlass.range_constexpr(ncol):+ col_idx_local = t0ScS_mn[0, col][1]+ # Convert to absolute column index+ global_col_idx = thr_col_offset + col_idx_local + n_block * self.tile_n++ batch_idx_ssa = utils.scalar_to_ssa(batch_idx, cutlass.Int32)+ head_idx_ssa = utils.scalar_to_ssa(head_idx, cutlass.Int32)+ q_idx_ssa = utils.scalar_to_ssa(+ tScS_mn[r, 0][0] + m_block * self.tile_m, cutlass.Int32+ )+ kv_idx_ssa = utils.scalar_to_ssa(+ thr_col_offset + t0ScS_mn[0, col][1] + n_block * self.tile_n,+ cutlass.Int32,+ )+ mask_value = mask_mod(+ batch_idx_ssa,+ head_idx_ssa,+ q_idx_ssa,+ kv_idx_ssa,+ aux_tensors,+ )+ cond = cutlass.Boolean(utils.ssa_to_scalar(mask_value))+ if const_expr(mask_seqlen):+ out_of_bounds = (global_row_idx >= self.seqlen_q) or (+ global_col_idx >= self.seqlen_k+ )+ if out_of_bounds:+ acc_S_mn[r, col] = -cutlass.Float32.inf+ else:+ acc_S_mn[r, col] = acc_S_mn[r, col] if cond else -cutlass.Float32.inf+ else:+ acc_S_mn[r, col] = acc_S_mn[r, col] if cond else -cutlass.Float32.inf++ else: # Causal or local+ if const_expr(not self.swap_AB):+ # If PackGQA, we split the work of compute divmod among threads in the same row+ threads_per_row = thr_mma.tv_layout_C.shape[0][0]+ mma_m_idx = None+ if const_expr(self.qhead_per_kvhead_packgqa != 1):+ assert not self.swap_AB, "swap_AB with PackGQA not supported yet"+ assert cute.arch.WARP_SIZE % threads_per_row == 0, (+ "threads_per_row must divide WARP_SIZE"+ )+ assert cute.size(acc_S_mn.shape[0]) <= threads_per_row+ tidx = thr_mma.thr_idx+ mma_m_idx = (+ m_block * self.tile_m + tScS_mn[tidx % threads_per_row, 0][0]+ ) // self.qhead_per_kvhead_packgqa+ causal_row_offset = (+ 1 + self.seqlen_k - n_block * self.tile_n - self.seqlen_q - thr_col_offset+ )+ if const_expr(mask_causal):+ r2p = const_expr(not self.swap_AB) # R2P trick, see apply_mask_sm100+ for r in cutlass.range(cute.size(tScS_mn.shape[0]), unroll_full=True):+ # get the column index limit based on current row. Only consider the row index, so the column index sets to 0.+ if const_expr(self.qhead_per_kvhead_packgqa == 1):+ row_idx = tScS_mn[r, 0][0] + m_block * self.tile_m+ else:+ row_idx = utils.shuffle_sync(+ mma_m_idx, r % threads_per_row, width=threads_per_row+ )+ col_limit_right = row_idx + causal_row_offset+ if const_expr(mask_seqlen):+ col_limit_right = cutlass.min(col_limit_right, seqlenk_col_limit)+ if const_expr(not r2p):+ # traverse column index.+ for c in cutlass.range(cute.size(tScS_mn.shape[1]), unroll_full=True):+ acc_S_mn[r, c] = (+ -Float32.inf+ if t0ScS_mn[0, c][1] >= col_limit_right+ else acc_S_mn[r, c]+ )+ else:+ mask_r2p(acc_S_mn[r, None], col_limit_right, arch=90, rank1=True)+ else: # Local+ local_row_offset_right = (+ causal_row_offset + self.window_size_right+ if const_expr(self.window_size_right is not None)+ else None+ )+ local_row_offset_left = (+ causal_row_offset - 1 - self.window_size_left+ if const_expr(self.window_size_left is not None)+ else None+ )+ for r in cutlass.range(cute.size(tScS_mn.shape[0]), unroll_full=True):+ if const_expr(self.qhead_per_kvhead_packgqa == 1):+ row_idx = tScS_mn[r, 0][0] + m_block * self.tile_m+ else:+ row_idx = utils.shuffle_sync(+ mma_m_idx, r % threads_per_row, width=threads_per_row+ )+ if const_expr(self.window_size_right is not None):+ col_limit_right = row_idx + local_row_offset_right+ if const_expr(mask_seqlen):+ col_limit_right = cutlass.min(col_limit_right, seqlenk_col_limit)+ else:+ col_limit_right = self.tile_n+ col_limit_left = (+ row_idx + local_row_offset_left+ if const_expr(self.window_size_left is not None)+ else 0+ )+ # if cute.arch.thread_idx()[0] == 128: cute.printf("n_block = {}, r = {}, row_idx = {}, causal_row_offset = {}, col_limit_right = {}, col_limit_left = {}", n_block, r, row_idx, causal_row_offset, col_limit_right, col_limit_left)+ # traverse column index.+ for c in cutlass.range(cute.size(tScS_mn.shape[1]), unroll_full=True):+ col_idx = t0ScS_mn[0, c][1]+ # only consider the column index, so the row index sets to 0.+ if col_idx >= col_limit_right or col_idx < col_limit_left:+ acc_S_mn[r, c] = -Float32.inf+ else: # swap_AB+ assert self.qhead_per_kvhead_packgqa == 1+ thr_row_offset = tScS_mn[0][ROW]+ causal_row_offset = (+ seqlenk_col_limit - self.seqlen_q + m_block * self.tile_m + thr_row_offset+ )+ if const_expr(mask_causal):+ for c in cutlass.range(cute.size(tScS_mn.shape[1]), unroll_full=True):+ col0 = t0ScS_mn[0, c][COL]+ # If col0 is beyond the column limit, we want to mask out the entire+ # column, by setting row limit to be self.tile_m.+ row_limit_top = (+ self.tile_m+ if col0 >= seqlenk_col_limit and mask_seqlen+ else col0 - causal_row_offset+ )+ for r in cutlass.range(cute.size(tScS_mn.shape[0]), unroll_full=True):+ acc_S_mn[r, c] = (+ -Float32.inf+ if t0ScS_mn[r, 0][ROW] < row_limit_top+ else acc_S_mn[r, c]+ )+ else:+ for c in cutlass.range(cute.size(tScS_mn.shape[1]), unroll_full=True):+ col0 = t0ScS_mn[0, c][COL]+ # If col0 is beyond the column limit, we want to mask out the entire+ # column, by setting row limit to be self.tile_m.+ row_limit_top = (+ self.tile_m+ if col0 >= seqlenk_col_limit+ else col0 - causal_row_offset - self.window_size_right+ )+ # TODO: do we need col_limit_sink?+ row_limit_bot = col0 - causal_row_offset + self.window_size_left+ for r in cutlass.range(cute.size(tScS_mn.shape[0]), unroll_full=True):+ row_idx = t0ScS_mn[r, 0][ROW]+ acc_S_mn[r, c] = (+ -Float32.inf+ if row_idx < row_limit_top or row_idx > row_limit_bot+ else acc_S_mn[r, c]+ )++ @cute.jit+ def apply_mask_sm100(+ self,+ acc_S: cute.Tensor,+ m_block: Int32,+ n_block: Int32,+ thr_mma: cute.TiledMma,+ thr_tmem_load: cute.TiledCopy,+ mask_seqlen: cutlass.Constexpr[bool],+ mask_causal: cutlass.Constexpr[bool],+ mask_local: cutlass.Constexpr[bool] = False,+ mask_mod: cutlass.Constexpr[Optional[Callable]] = None,+ batch_idx: Int32 = None,+ head_idx: Int32 = None,+ aux_tensors: Optional[list] = None,+ ) -> None:+ assert not (mask_causal and mask_local), "mask_causal and mask_local cannot be both True"+ acc_shape = (self.tile_m, self.tile_n)+ cS = cute.make_identity_tensor(acc_shape if not self.swap_AB else acc_shape[::-1])+ tScS = thr_mma.partition_C(cS)+ tScS_t2r = thr_tmem_load.partition_D(tScS)+ # To handle edge cases of completely masked out rows where n_block_max = 0,+ # we treat negative n_blocks as 0th n_block+ # TODO: find more transparent solution+ if n_block < 0:+ n_block = 0+ seqlenk_col_limit = self.seqlen_k - n_block * self.tile_n+ r2p = True+ if const_expr(not mask_causal and not mask_local and mask_mod is None):+ if const_expr(mask_seqlen):+ if const_expr(not r2p):+ for i in cutlass.range(cute.size(tScS_t2r.shape), unroll_full=True):+ # if tScS_t2r[i][1] >= seqlenk_col_limit:+ # acc_S[i] = -Float32.inf+ # For some reason the 2 lines above generate really bad SASS+ acc_S[i] = -Float32.inf if tScS_t2r[i][1] >= seqlenk_col_limit else acc_S[i]+ else:+ mask_r2p(acc_S, seqlenk_col_limit, arch=100, rank1=True)++ elif const_expr(not mask_causal and not mask_local and mask_mod is not None):+ # Block sparse case w/ mask_mod+ batch_idx_ssa = utils.scalar_to_ssa(batch_idx, cutlass.Int32)+ head_idx_ssa = utils.scalar_to_ssa(head_idx, cutlass.Int32)+ row_coord_first = tScS_t2r[0][0]+ global_row = row_coord_first + m_block * self.tile_m+ if const_expr(self.qhead_per_kvhead_packgqa != 1):+ mask_row = global_row // self.qhead_per_kvhead_packgqa+ else:+ mask_row = global_row+ mask_row_ssa = utils.scalar_to_ssa(mask_row, cutlass.Int32)++ ncol = const_expr(cute.size(tScS_t2r.shape))+ for i in cutlass.range_constexpr(ncol):+ col_coord = tScS_t2r[i][1] if not self.swap_AB else tScS_t2r[i][0]+ global_col = col_coord + n_block * self.tile_n+ mask_value = mask_mod(+ batch_idx_ssa,+ head_idx_ssa,+ mask_row_ssa,+ utils.scalar_to_ssa(global_col, cutlass.Int32),+ aux_tensors,+ )+ cond = cutlass.Boolean(utils.ssa_to_scalar(mask_value))+ acc_S[i] = acc_S[i] if cond else -Float32.inf+ if const_expr(mask_seqlen):+ out_of_bounds = (global_row >= self.seqlen_q) or (global_col >= self.seqlen_k)+ acc_S[i] = -Float32.inf if out_of_bounds else acc_S[i]++ else: # Causal or local+ causal_row_offset = 1 + self.seqlen_k - n_block * self.tile_n - self.seqlen_q+ row_idx = tScS_t2r[0][0] + m_block * self.tile_m+ if const_expr(self.qhead_per_kvhead_packgqa != 1):+ row_idx = row_idx // self.qhead_per_kvhead_packgqa+ if const_expr(mask_causal):+ col_limit_right = row_idx + causal_row_offset+ if const_expr(mask_seqlen):+ col_limit_right = cutlass.min(col_limit_right, seqlenk_col_limit)+ # if cute.arch.thread_idx()[0] % 32 == 0:+ # cute.printf("tidx = %d, tidx tmem = %d, row_idx = %d, col_limit_right = %d, causal_row_offset = %d\n", cute.arch.thread_idx()[0], thr_tmem_load.thr_idx, row_idx, col_limit_right, causal_row_offset)+ ncol = const_expr(cute.size(tScS_t2r.shape))+ if const_expr(not r2p):+ for i in cutlass.range(ncol, unroll_full=True):+ acc_S[i] = -Float32.inf if tScS_t2r[i][1] >= col_limit_right else acc_S[i]+ else:+ mask_r2p(acc_S, col_limit_right, arch=100, rank1=True)+ else:+ local_row_offset_right = (+ causal_row_offset + self.window_size_right+ if const_expr(self.window_size_right is not None)+ else None+ )+ local_row_offset_left = (+ causal_row_offset - 1 - self.window_size_left+ if const_expr(self.window_size_left is not None)+ else None+ )+ if const_expr(self.window_size_right is not None):+ col_limit_right = row_idx + local_row_offset_right+ if const_expr(mask_seqlen):+ col_limit_right = cutlass.min(col_limit_right, seqlenk_col_limit)+ else:+ col_limit_right = self.tile_n+ col_limit_left = (+ row_idx + local_row_offset_left+ if const_expr(self.window_size_left is not None)+ else 0+ )+ # if cute.arch.thread_idx()[0] == 0 or cute.arch.thread_idx()[0] == 128: cute.printf("m_block = {}, n_block = {}, row_idx = {}, causal_row_offset = {}, col_limit_right = {}, col_limit_left = {}", m_block, n_block, row_idx, causal_row_offset, col_limit_right, col_limit_left)+ for i in cutlass.range(cute.size(tScS_t2r.shape), unroll_full=True):+ col_idx = tScS_t2r[i][1]+ acc_S[i] = (+ -Float32.inf+ if col_idx >= col_limit_right or col_idx < col_limit_left+ else acc_S[i]+ )++ @cute.jit+ def apply_mask_sm100_transposed(+ self,+ acc_S: cute.Tensor,+ tScS_t2r: cute.Tensor,+ t0ScS_t2r: cute.Tensor,+ m_block: cutlass.Int32,+ n_block: cutlass.Int32,+ mask_seqlen: cutlass.Constexpr,+ mask_causal: cutlass.Constexpr,+ mask_local: cutlass.Constexpr,+ ) -> None:+ """+ Backward pass: mask S = K @ Q.T where n_block tiles seqlen_k and m_block tiles seqlen_q.+ """+ assert not (mask_causal and mask_local), "mask_causal and mask_local cannot be both True"+ ROW = 0 if const_expr(not self.swap_AB) else 1+ COL = 1 if const_expr(not self.swap_AB) else 0+ thr_col_offset = tScS_t2r[0][COL]+ seqlenk_col_limit = self.seqlen_k - n_block * self.tile_n - thr_col_offset+ if const_expr(not mask_causal and not mask_local):+ if const_expr(mask_seqlen):+ if t0ScS_t2r[0][COL] >= seqlenk_col_limit:+ for i in cutlass.range(cute.size(acc_S.shape), unroll_full=True):+ acc_S[i] = -cutlass.Float32.inf+ else: # Causal or local+ thr_row_offset = tScS_t2r[0][ROW]+ causal_row_offset = (+ seqlenk_col_limit - self.seqlen_q + m_block * self.tile_m + thr_row_offset+ )+ if const_expr(mask_causal):+ col0 = t0ScS_t2r[0][COL]+ row_limit_top = col0 - causal_row_offset+ # tidx = cute.arch.thread_idx()[0] % 256+ # if tidx < 32:+ # cute.printf("tidx = {}, {} {}, {} {}, col0 = {}", tidx, tScS_t2r[0][0], tScS_t2r[0][1], tScS_t2r[1][0], tScS_t2r[1][1], col0)+ if const_expr(mask_seqlen):+ # If col is beyond the column limit, we want to mask out the entire+ # column, by setting row limit to be self.tile_m.+ if t0ScS_t2r[0][COL] >= seqlenk_col_limit:+ row_limit_top = self.tile_m+ r2p = True+ if const_expr(not r2p):+ for i in cutlass.range(cute.size(acc_S.shape), unroll_full=True):+ acc_S[i] = (+ -cutlass.Float32.inf if t0ScS_t2r[i][ROW] < row_limit_top else acc_S[i]+ )+ else:+ num_rep = cute.size(tScS_t2r, mode=[0]) # 16 or 32+ mask_r2p_transposed(acc_S, row_limit_top, num_rep)+ else:+ assert False, "Local masking isn't supported yet"diff --git a/src/nvfp4_decode_kernel/_fa4/mma_sm100_desc.py b/src/nvfp4_decode_kernel/_fa4/mma_sm100_desc.pynew file mode 100644index 0000000..abf3b21--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/mma_sm100_desc.py@@ -0,0 +1,400 @@+# Copyright (c) 2025, Tri Dao.+# Ported Cutlass code from C++ to Python:+# https://github.com/NVIDIA/cutlass/blob/main/include/cute/arch/mma_sm100_desc.hpp+# https://github.com/NVIDIA/cutlass/blob/main/include/cute/atom/mma_traits_sm100.hpp++from enum import IntEnum++import cutlass+import cutlass.cute as cute++# ---------------------------------------------------------------------------+# Enumerations that match the HW encodings (values MUST stay identical)+# ---------------------------------------------------------------------------+++class Major(IntEnum): # matrix “layout” in the ISA docs+ K = 0+ MN = 1+++class ScaleIn(IntEnum): # negate flags+ One = 0+ Neg = 1+++class Saturate(IntEnum):+ False_ = 0+ True_ = 1+++class CFormat(IntEnum): # 2-bit field (bits 4-5)+ F16 = 0+ F32 = 1+ S32 = 2+++class F16F32Format(IntEnum): # 3-bit field (A/B element type)+ F16 = 0+ BF16 = 1+ TF32 = 2+++class S8Format(IntEnum):+ UINT8 = 0+ INT8 = 1+++class MXF8F6F4Format(IntEnum):+ E4M3 = 0+ E5M2 = 1+ E2M3 = 3+ E3M2 = 4+ E2M1 = 5+++class MaxShift(IntEnum):+ NoShift = 0+ MaxShift8 = 1+ MaxShift16 = 2+ MaxShift32 = 3+++class ScaleFormat(IntEnum):+ UE4M3 = 0+ UE8M0 = 1+++# ---------------------------------------------------------------------------+# CUTLASS-type → encoding helpers+# ---------------------------------------------------------------------------+++def to_UMMA_format(cutlass_type) -> int:+ """+ Map a CUTLASS scalar class to the 3-bit encoding for Matrix A/B.+ """+ if cutlass_type is cutlass.Int8:+ return S8Format.INT8+ # Unsigned 8-bit (if available in your CUTLASS build)+ if cutlass_type is cutlass.Uint8:+ return S8Format.UINT8+ # FP-16 / BF-16+ if cutlass_type is cutlass.Float16:+ return F16F32Format.F16+ if cutlass_type is cutlass.BFloat16:+ return F16F32Format.BF16+ # TensorFloat-32 (8-bit exponent, 10-bit mantissa packed in 19 bits)+ if cutlass_type is cutlass.TFloat32:+ return F16F32Format.TF32+ # Float-8 / Float-6 / Float-4 – add whenever CUTLASS exposes them+ if cutlass_type is cutlass.Float8E4M3FN:+ return MXF8F6F4Format.E4M3+ if cutlass_type is cutlass.Float8E5M2:+ return MXF8F6F4Format.E5M2+ if cutlass_type is cutlass.Float4E2M1FN:+ return MXF8F6F4Format.E2M1+ raise TypeError(f"Unsupported CUTLASS scalar type for A/B: {cutlass_type!r}")+++def to_C_format(cutlass_type) -> int:+ """+ Map a CUTLASS scalar class to the 2-bit accumulator encoding.+ """+ if cutlass_type is cutlass.Float16:+ return CFormat.F16+ if cutlass_type is cutlass.Float32:+ return CFormat.F32+ if cutlass_type is cutlass.Int32:+ return CFormat.S32+ raise TypeError(f"Unsupported CUTLASS scalar type for accumulator: {cutlass_type!r}")+++def to_ScaleFormat(cutlass_type) -> int:+ """+ Map a CUTLASS scalar class to the scale format encoding.+ """+ # For now, we'll use UE4M3 as default for FP4 scale factors+ # This may need to be adjusted based on actual scale factor types used+ if cutlass_type is cutlass.Float8E4M3FN:+ return ScaleFormat.UE4M3+ elif cutlass_type is cutlass.Float8E8M0FNU:+ return ScaleFormat.UE8M0+ raise TypeError(f"Unsupported CUTLASS scalar type for scale factor: {cutlass_type!r}")+++# ---------------------------------------------------------------------------+# The constructor – accepts only CUTLASS scalar classes+# ---------------------------------------------------------------------------+++def make_instr_desc(+ a_type, # CUTLASS scalar class, e.g. cutlass.Int8+ b_type,+ c_type,+ M: int, # 64, 128 or 256+ N: int, # 8 … 256 (multiple of 8)+ a_major: Major,+ b_major: Major,+ a_neg: ScaleIn = ScaleIn.One,+ b_neg: ScaleIn = ScaleIn.One,+ c_sat: Saturate = Saturate.False_,+ is_sparse: bool = False,+ max_shift: MaxShift = MaxShift.NoShift,+) -> int:+ """+ Build the 32-bit instruction descriptor for Blackwell MMA.+ All matrix/accumulator **types must be CUTLASS scalar classes** –+ passing integers is forbidden.+ """+ # --- encode element formats -------------------------------------------------+ a_fmt = int(to_UMMA_format(a_type))+ b_fmt = int(to_UMMA_format(b_type))+ c_fmt = int(to_C_format(c_type))++ # --- range checks on M/N -----------------------------------------------------+ if M not in (64, 128, 256):+ raise ValueError("M must be 64, 128 or 256")+ if N < 8 or N > 256 or (N & 7):+ raise ValueError("N must be a multiple of 8 in the range 8…256")++ m_dim = M >> 4 # 5-bit field+ n_dim = N >> 3 # 6-bit field++ # fmt: off+ # --- pack the bit-fields -----------------------------------------------------+ desc = 0+ desc |= (0 & 0x3) << 0 # sparse_id2 (always 0 here)+ desc |= (int(is_sparse) & 0x1) << 2 # sparse_flag+ desc |= (int(c_sat) & 0x1) << 3 # saturate+ desc |= (c_fmt & 0x3) << 4 # c_format+ desc |= (a_fmt & 0x7) << 7 # a_format+ desc |= (b_fmt & 0x7) << 10 # b_format+ desc |= (int(a_neg) & 0x1) << 13 # a_negate+ desc |= (int(b_neg) & 0x1) << 14 # b_negate+ desc |= (int(a_major) & 0x1) << 15 # a_major+ desc |= (int(b_major) & 0x1) << 16 # b_major+ desc |= (n_dim & 0x3F) << 17 # n_dim (6 bits)+ desc |= (m_dim & 0x1F) << 24 # m_dim (5 bits)+ desc |= (int(max_shift) & 0x3) << 30 # max_shift (2 bits)+ # fmt: on++ return desc & 0xFFFF_FFFF # ensure 32-bit result++def make_instr_desc_block_scaled(+ a_type,+ b_type,+ c_type,+ sf_type, # Scale factor type+ M: int,+ N: int,+ a_major: Major,+ b_major: Major,+ a_neg: ScaleIn = ScaleIn.One,+ b_neg: ScaleIn = ScaleIn.One,+ is_sparse: bool = False,+) -> int:+ """+ Build the 32-bit instruction descriptor for Blackwell block-scaled MMA.+ All matrix/accumulator/scale factor **types must be CUTLASS scalar classes**.+ """+ # --- encode element formats -------------------------------------------------+ a_fmt = int(to_UMMA_format(a_type))+ b_fmt = int(to_UMMA_format(b_type))+ sf_fmt = int(to_ScaleFormat(sf_type))+ # Packed E2M1 is MXF4Format::E2M1 = 1 under kind::mxf4 (CUTLASS's+ # to_UMMAFormat<float_e2m1_t>); the 5 above is the f8f6f4 code, which+ # kind::mxf4 rejects as an illegal instruction. NVFP4 keeps 5, which its+ # kind accepts, so its kernels stay byte for byte what they were.+ if sf_type is cutlass.Float8E8M0FNU and a_type is cutlass.Float4E2M1FN:+ a_fmt = b_fmt = 1++ # --- range checks on M/N -----------------------------------------------------+ if M not in (64, 128, 256):+ raise ValueError("M must be 64, 128 or 256")+ if N < 8 or N > 256 or (N & 7):+ raise ValueError("N must be a multiple of 8 in the range 8…256")++ m_dim = M >> 4 # 5-bit field+ n_dim = N >> 3 # 6-bit field++ # --- pack the bit-fields -----------------------------------------------------+ # Based on InstrDescriptorBlockScaled structure:+ # sparse_id2_: bits [0, 2) - 2 bits+ # sparse_flag_: bit [2, 3) - 1 bit+ # reserved: bit [3, 4) - 1 bit+ # b_sf_id_: bits [4, 6) - 2 bits (set to 0 initially, updated at runtime)+ # reserved: bit [6, 7) - 1 bit+ # a_format_: bits [7, 10) - 3 bits+ # b_format_: bits [10, 13) - 3 bits+ # a_negate_: bit [13, 14) - 1 bit+ # b_negate_: bit [14, 15) - 1 bit+ # a_major_: bit [15, 16) - 1 bit+ # b_major_: bit [16, 17) - 1 bit+ # n_dim_: bits [17, 23) - 6 bits+ # scale_format_: bit [23, 24) - 1 bit+ # m_dim_: bits [24, 29) - 5 bits+ # a_sf_id_: bits [29, 31) - 2 bits (set to 0 initially, updated at runtime)+ # k_size_: bit [31, 32) - 1 bit. MXF8F6F4Format: 0=[dense: K32, sparse: K64]++ desc = 0+ desc |= (0 & 0x3) << 0 # sparse_id2 (always 0 here)+ desc |= (int(is_sparse) & 0x1) << 2 # sparse_flag+ desc |= (0 & 0x1) << 3 # reserved+ desc |= (0 & 0x3) << 4 # b_sf_id+ desc |= (0 & 0x1) << 6 # reserved+ desc |= (a_fmt & 0x7) << 7 # a_format+ desc |= (b_fmt & 0x7) << 10 # b_format+ desc |= (int(a_neg) & 0x1) << 13 # a_negate+ desc |= (int(b_neg) & 0x1) << 14 # b_negate+ desc |= (int(a_major) & 0x1) << 15 # a_major+ desc |= (int(b_major) & 0x1) << 16 # b_major+ desc |= (n_dim & 0x3F) << 17 # n_dim (6 bits)+ desc |= (sf_fmt & 0x1) << 23 # scale_format+ desc |= (m_dim & 0x1F) << 24 # m_dim (5 bits)+ desc |= (0 & 0x3) << 29 # a_sf_id+ desc |= (0 & 0x1) << 31 # k_size++ return desc & 0xFFFF_FFFF # ensure 32-bit result+++def mma_op_to_idesc(op: cute.nvgpu.tcgen05.mma.MmaOp):+ # FP4 takes the block-scaled descriptor, whose scale format is the op's:+ # E4M3 for NVFP4, E8M0 for MXFP4.+ if op.a_dtype is cutlass.Float4E2M1FN:+ return make_instr_desc_block_scaled(+ op.a_dtype,+ op.b_dtype,+ op.acc_dtype,+ op.sf_dtype,+ op.shape_mnk[0],+ op.shape_mnk[1],+ Major.K if op.a_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K else Major.MN,+ Major.K if op.b_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K else Major.MN,+ )+ return make_instr_desc(+ op.a_dtype,+ op.b_dtype,+ op.acc_dtype,+ op.shape_mnk[0],+ op.shape_mnk[1],+ Major.K if op.a_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K else Major.MN,+ Major.K if op.b_major_mode == cute.nvgpu.tcgen05.mma.OperandMajorMode.K else Major.MN,+ )+++class LayoutType(IntEnum): # occupies the top-3 bits [61:64)+ SWIZZLE_NONE = 0 # (a.k.a. “INTERLEAVE” in older docs)+ SWIZZLE_128B_BASE32B = 1+ SWIZZLE_128B = 2+ SWIZZLE_64B = 4+ SWIZZLE_32B = 6+ # values 3,5,7 are reserved / illegal for UMMA+++# ---------------------------------------------------------------------------+# Helpers – figure out the SWIZZLE_* family from the tensor layout+# ---------------------------------------------------------------------------+++def _layout_type(swizzle: cute.Swizzle) -> LayoutType:+ # No idea what the right way to get B, M, S is – so we're just parsing it from the __str__+ # Swizzle string has the form "S<B,M,S>"+ swz_str = str(swizzle)+ inside = swz_str[swz_str.index("<") + 1 : swz_str.index(">")] # '3,4,3'+ B, M, S = [int(x) for x in inside.split(",")] # [3, 4, 3]++ if M == 4: # Swizzle<*,4,3>+ if S != 3:+ raise ValueError("Unexpected swizzle shift – want S==3 for M==4")+ return {+ 0: LayoutType.SWIZZLE_NONE,+ 1: LayoutType.SWIZZLE_32B,+ 2: LayoutType.SWIZZLE_64B,+ 3: LayoutType.SWIZZLE_128B,+ }[B] # KeyError ⇒ invalid B→ raise+ if M == 5: # Swizzle<2,5,2> (the only legal triple for M==5)+ if (B, S) != (2, 2):+ raise ValueError("Only Swizzle<2,5,2> supported for 128B_BASE32B")+ return LayoutType.SWIZZLE_128B_BASE32B++ # Any other (M,B,S) triple is not a UMMA-legal shared-memory layout+ raise ValueError("Unsupported swizzle triple for UMMA smem descriptor")+++def make_smem_desc_base(layout: cute.Layout, swizzle: cute.Swizzle, major: Major) -> int:+ """+ Convert a 2-D *shared-memory* Cute layout into the Blackwell 64-bit+ smem-descriptor, without the smem start address.+ layout must correspond to layout of an uint128 tensor.+ """+ # ------------------------------------------------------------------ meta+ layout_type = _layout_type(swizzle) # resolve SWIZZLE_* family++ VERSION = 1 # bits 46–47+ LBO_MODE = 0 # bit 52+ BASE_OFFSET = 0 # bits 49–51 (CUTLASS always 0)++ # ---------------------------------------------------------- strides (units: uint128_t = 16 B)+ swizzle_atom_mn_size = {+ LayoutType.SWIZZLE_NONE: 1,+ LayoutType.SWIZZLE_32B: 2,+ LayoutType.SWIZZLE_64B: 4,+ LayoutType.SWIZZLE_128B: 8,+ LayoutType.SWIZZLE_128B_BASE32B: 8,+ }[layout_type]++ if major is Major.MN:+ swizzle_atom_k_size = 4 if layout_type is LayoutType.SWIZZLE_128B_BASE32B else 8+ canonical_layout = cute.logical_divide(layout, (swizzle_atom_mn_size, swizzle_atom_k_size))+ if not cute.is_congruent(canonical_layout, ((1, 1), (1, 1))):+ raise ValueError("Not a canonical UMMA_MN Layout: Expected profile failure.")+ stride_00 = canonical_layout.stride[0][0]+ if layout_type is not LayoutType.SWIZZLE_NONE and stride_00 != 1:+ raise ValueError("Not a canonical UMMA_MN Layout: Expected stride failure.")+ stride_10 = canonical_layout.stride[1][0]+ if stride_10 != swizzle_atom_mn_size:+ raise ValueError("Not a canonical UMMA_MN Layout: Expected stride failure.")+ stride_01, stride_11 = canonical_layout.stride[0][1], canonical_layout.stride[1][1]+ if layout_type is LayoutType.SWIZZLE_NONE:+ stride_byte_offset, leading_byte_offset = stride_01, stride_11+ else:+ stride_byte_offset, leading_byte_offset = stride_11, stride_01+ else:+ if layout_type == LayoutType.SWIZZLE_128B_BASE32B:+ raise ValueError("SWIZZLE_128B_BASE32B is invalid for Major-K")+ if not cute.size(layout.shape[0]) % 8 == 0:+ raise ValueError("Not a canonical UMMA_K Layout: Expected MN-size multiple of 8.")+ canonical_layout = cute.logical_divide(layout, (8, 2))+ if not cute.is_congruent(canonical_layout, ((1, 1), (1, 1))):+ raise ValueError("Not a canonical UMMA_K Layout: Expected profile failure.")+ stride_00 = canonical_layout.stride[0][0]+ if stride_00 != swizzle_atom_mn_size:+ raise ValueError("Not a canonical UMMA_K Layout: Expected stride failure.")+ stride_10 = canonical_layout.stride[1][0]+ if layout_type is not LayoutType.SWIZZLE_NONE and stride_10 != 1:+ raise ValueError("Not a canonical UMMA_K Layout: Expected stride failure.")+ stride_01 = canonical_layout.stride[0][1]+ stride_byte_offset, leading_byte_offset = stride_01, stride_10++ # ------------------------------------------------------------------ pack+ desc = 0+ # leading_byte_offset_ [16:30)+ desc |= (leading_byte_offset & 0x3FFF) << 16+ # stride_byte_offset_ [32:46)+ desc |= (stride_byte_offset & 0x3FFF) << 32+ # version_ [46:48)+ desc |= (VERSION & 0x3) << 46+ # base_offset_ [49:52)+ desc |= (BASE_OFFSET & 0x7) << 49+ # lbo_mode_ [52:53)+ desc |= (LBO_MODE & 0x1) << 52+ # layout_type_ [61:64)+ desc |= (int(layout_type) & 0x7) << 61++ return desc & 0xFFFF_FFFF_FFFF_FFFF # force 64-bit width+++def make_smem_desc_start_addr(start_addr: cute.Pointer) -> cutlass.Int32:+ # 14 bits, remove 4 LSB (bits 0-13 in desc)+ return (start_addr.toint() & 0x3FFFF) >> 4diff --git a/src/nvfp4_decode_kernel/_fa4/pack_gqa.py b/src/nvfp4_decode_kernel/_fa4/pack_gqa.pynew file mode 100644index 0000000..2bd4929--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/pack_gqa.py@@ -0,0 +1,221 @@+# Copyright (c) 2025, Tri Dao.++import math+import operator++import cutlass+import cutlass.cute as cute++from . import utils as utils+++class PackGQA:+ def __init__(+ self,+ m_block_size: cutlass.Constexpr[int],+ head_dim_padded: cutlass.Constexpr[int],+ check_hdim_oob: cutlass.Constexpr[bool],+ qhead_per_kvhead: cutlass.Constexpr[bool],+ ):+ self.m_block_size = m_block_size+ self.head_dim_padded = head_dim_padded+ self.check_hdim_oob = check_hdim_oob+ self.qhead_per_kvhead = qhead_per_kvhead++ @cute.jit+ def compute_ptr(+ self,+ tensor: cute.Tensor,+ cRows: cute.Tensor,+ tidx: cutlass.Int32,+ block: cutlass.Int32,+ threads_per_row: cutlass.Constexpr[int],+ num_threads: cutlass.Constexpr[int],+ row_stride: cutlass.Constexpr[int] = 1,+ ):+ """Address of each row this thread will move.++ ``row_stride`` describes a tile whose query rows have been repeated, so+ that row ``r * row_stride`` carries head ``r``. It is only meaningful+ with a single query position, where the block index is zero and the+ sequence coordinate is always zero.++ ``tensor`` is indexed by either the hierarchical (head, seqlen) row+ mode or, for a single query position, a flat head axis; the flat form+ takes the head index alone.+ """+ num_ptr_per_thread = cute.ceil_div(cute.size(cRows), threads_per_row)+ tPrPtr = cute.make_rmem_tensor(num_ptr_per_thread, cutlass.Int64)+ for i in cutlass.range_constexpr(num_ptr_per_thread):+ row = i * num_threads + cRows[tidx % threads_per_row][0]+ if cutlass.const_expr(row_stride > 1):+ h_idx = row // row_stride+ m_idx = 0+ else:+ idx = block * self.m_block_size + row+ m_idx = idx // self.qhead_per_kvhead+ h_idx = idx - m_idx * self.qhead_per_kvhead+ if cutlass.const_expr(cute.rank(tensor.shape[0]) == 1):+ tPrPtr[i] = utils.elem_pointer(tensor, (h_idx,)).toint()+ else:+ tPrPtr[i] = utils.elem_pointer(tensor, ((h_idx, m_idx),)).toint()+ return tPrPtr++ @cute.jit+ def load_Q(+ self,+ mQ: cute.Tensor, # ((qhead_per_kvhead, seqlen_q), headdim)+ sQ: cute.Tensor, # (m_block_size, head_dim_padded)+ gmem_tiled_copy: cute.TiledCopy,+ tidx: cutlass.Int32,+ block: cutlass.Int32,+ seqlen: cutlass.Int32,+ ):+ gmem_thr_copy = gmem_tiled_copy.get_slice(tidx)+ cQ = cute.make_identity_tensor((self.m_block_size, self.head_dim_padded))+ tQsQ = gmem_thr_copy.partition_D(sQ)+ tQcQ = gmem_thr_copy.partition_S(cQ)+ t0QcQ = gmem_thr_copy.get_slice(0).partition_S(cQ)+ tQpQ = utils.predicate_k(tQcQ, limit=mQ.shape[1])+ tQcQ_row = tQcQ[0, None, 0]+ threads_per_row = gmem_tiled_copy.layout_tv_tiled.shape[0][0]+ assert cute.arch.WARP_SIZE % threads_per_row == 0, "threads_per_row must divide WARP_SIZE"+ num_threads = gmem_tiled_copy.size+ tPrQPtr = self.compute_ptr(mQ[None, 0], tQcQ_row, tidx, block, threads_per_row, num_threads)+ for m in cutlass.range_constexpr(cute.size(tQsQ.shape[1])):+ q_ptr_i64 = utils.shuffle_sync(+ tPrQPtr[m // threads_per_row], m % threads_per_row, width=threads_per_row+ )+ q_gmem_ptr = cute.make_ptr(+ mQ.element_type, q_ptr_i64, cute.AddressSpace.gmem, assumed_align=16+ )+ if (+ t0QcQ[0, m, 0][0]+ < seqlen * self.qhead_per_kvhead - block * self.m_block_size - tQcQ_row[0][0]+ ):+ mQ_cur = cute.make_tensor(q_gmem_ptr, (self.head_dim_padded,))+ elems_per_load = cute.size(tQsQ.shape[0][0])+ mQ_cur_copy = cute.tiled_divide(mQ_cur, (elems_per_load,))+ for k in cutlass.range_constexpr(cute.size(tQsQ.shape[2])):+ ki = tQcQ[0, 0, k][1] // elems_per_load+ cute.copy(+ gmem_thr_copy,+ mQ_cur_copy[None, ki],+ tQsQ[None, m, k],+ pred=tQpQ[None, m, k] if cutlass.const_expr(self.check_hdim_oob) else None,+ )+ # We don't need to clear the sQ smem tiles since we'll only write out the valid outputs++ @cute.jit+ def store_LSE(+ self,+ mLSE: cute.Tensor, # (qhead_per_kvhead, seqlen_q)+ tLSErLSE: cute.Tensor, # (m_block_size, head_dim_padded)+ tiled_mma: cute.TiledMma,+ tidx: cutlass.Int32,+ block: cutlass.Int32,+ seqlen: cutlass.Int32,+ ):+ thr_mma = tiled_mma.get_slice(tidx)+ caccO = cute.make_identity_tensor((self.m_block_size, self.head_dim_padded))+ taccOcO = thr_mma.partition_C(caccO)+ taccOcO_row = utils.make_acc_tensor_mn_view(taccOcO)[None, 0]+ assert cute.size(tLSErLSE) == cute.size(taccOcO_row)+ threads_per_row = tiled_mma.tv_layout_C.shape[0][0]+ assert cute.arch.WARP_SIZE % threads_per_row == 0, "threads_per_row must divide WARP_SIZE"+ assert cute.size(tLSErLSE) <= threads_per_row+ num_threads = tiled_mma.size+ tPrLSEPtr = self.compute_ptr(mLSE, taccOcO_row, tidx, block, threads_per_row, num_threads)+ for m in cutlass.range_constexpr(cute.size(tLSErLSE)):+ lse_ptr_i64 = utils.shuffle_sync(+ tPrLSEPtr[m // threads_per_row],+ m % threads_per_row,+ width=threads_per_row,+ )+ lse_gmem_ptr = cute.make_ptr(+ mLSE.element_type, lse_ptr_i64, cute.AddressSpace.gmem, assumed_align=4+ )+ row = block * self.m_block_size + taccOcO_row[m][0]+ # Only the thread corresponding to column 0 writes out the lse to gmem+ if taccOcO[0][1] == 0 and row < seqlen * self.qhead_per_kvhead:+ mLSE_copy = cute.make_tensor(lse_gmem_ptr, (1,))+ mLSE_copy[0] = tLSErLSE[m]++ @cute.jit+ def store_O(+ self,+ mO: cute.Tensor, # ((qhead_per_kvhead, seqlen_q), headdim)+ tOrO: cute.Tensor, # (m_block_size, head_dim_padded) split across threads according to gmem_tiled_copy+ gmem_tiled_copy: cute.TiledCopy,+ tidx: cutlass.Int32,+ block: cutlass.Int32,+ seqlen: cutlass.Int32,+ max_rows: cutlass.Constexpr = None,+ stage_from_smem: cutlass.Constexpr[bool] = False,+ row_stride: cutlass.Constexpr[int] = 1,+ ):+ """Store one output tile, one row-step at a time.++ ``max_rows`` bounds, at compile time, how many rows of this tile the+ runtime predicate below can admit; the caller must guarantee+ ``seqlen * qhead_per_kvhead - block * m_block_size <= max_rows``. Row+ steps past that bound would store nothing, so the loop simply stops.++ ``stage_from_smem`` makes ``tOrO`` a shared-memory tensor of the same+ partitioned shape instead of a register fragment, staged into registers+ one row-step at a time. Materializing the whole tile up front costs 256+ registers per thread, which spills when this runs on a single warp that+ has already dropped to a small register budget.++ ``row_stride`` says the tile's query rows were repeated on the way in,+ so head ``r`` is at row ``r * row_stride`` and the rows in between are+ duplicates that must not be stored. Every row of the tile is then+ reachable, so the caller's ``max_rows`` must cover the whole tile.+ """+ gmem_thr_copy = gmem_tiled_copy.get_slice(tidx)+ cO = cute.make_identity_tensor((self.m_block_size, self.head_dim_padded))+ tOcO = gmem_thr_copy.partition_S(cO)+ t0OcO = gmem_thr_copy.get_slice(0).partition_S(cO)+ tOpO = utils.predicate_k(tOcO, limit=mO.shape[1])+ tOcO_row = tOcO[0, None, 0]+ threads_per_row = gmem_tiled_copy.layout_tv_tiled.shape[0][0]+ assert cute.arch.WARP_SIZE % threads_per_row == 0, "threads_per_row must divide WARP_SIZE"+ num_threads = gmem_tiled_copy.size+ tPrOPtr = self.compute_ptr(+ mO[None, 0], tOcO_row, tidx, block, threads_per_row, num_threads, row_stride+ )+ num_row_steps = cute.size(tOrO.shape[1])+ if cutlass.const_expr(max_rows is not None):+ rows_per_step = num_threads // threads_per_row+ num_row_steps = min(num_row_steps, -(-max_rows // rows_per_step))+ for m in cutlass.range_constexpr(num_row_steps):+ o_ptr_i64 = utils.shuffle_sync(+ tPrOPtr[m // threads_per_row], m % threads_per_row, width=threads_per_row+ )+ o_gmem_ptr = cute.make_ptr(+ mO.element_type, o_ptr_i64, cute.AddressSpace.gmem, assumed_align=16+ )+ if cutlass.const_expr(stage_from_smem):+ tOrO_m = cute.make_fragment_like(tOrO[None, m, None], mO.element_type)+ cute.autovec_copy(tOrO[None, m, None], tOrO_m)+ else:+ tOrO_m = tOrO[None, m, None]+ if cutlass.const_expr(row_stride > 1):+ row_is_live = (t0OcO[0, m, 0][0] + tOcO_row[0][0]) % row_stride == 0+ else:+ row_is_live = (+ t0OcO[0, m, 0][0]+ < seqlen * self.qhead_per_kvhead - block * self.m_block_size - tOcO_row[0][0]+ )+ if row_is_live:+ mO_cur = cute.make_tensor(o_gmem_ptr, (self.head_dim_padded,))+ elems_per_load = cute.size(tOrO.shape[0][0])+ mO_cur_copy = cute.tiled_divide(mO_cur, (elems_per_load,))+ for k in cutlass.range_constexpr(cute.size(tOrO.shape[2])):+ ki = tOcO[0, 0, k][1] // elems_per_load+ cute.copy(+ gmem_thr_copy,+ tOrO_m[None, k],+ mO_cur_copy[None, ki],+ pred=tOpO[None, m, k] if cutlass.const_expr(self.check_hdim_oob) else None,+ )diff --git a/src/nvfp4_decode_kernel/_fa4/paged_kv.py b/src/nvfp4_decode_kernel/_fa4/paged_kv.pynew file mode 100644index 0000000..fb71012--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/paged_kv.py@@ -0,0 +1,176 @@+from typing import Type+from dataclasses import dataclass++import cutlass+import cutlass.cute as cute+from cutlass.cute.nvgpu import cpasync+from cutlass import Int32, const_expr++from . import utils+from .fast_math import FastDivmod+from .cute_dsl_utils import ParamsBase+++@dataclass+class PagedKVManager(ParamsBase):+ mPageTable: cute.Tensor+ mK_paged: cute.Tensor+ mV_paged: cute.Tensor+ thread_idx: Int32++ page_size_divmod: FastDivmod+ seqlen_k: Int32+ leftpad_k: Int32+ n_block_size: Int32+ num_threads: cutlass.Constexpr[Int32]+ head_dim_padded: cutlass.Constexpr[Int32]+ head_dim_v_padded: cutlass.Constexpr[Int32]++ gmem_threads_per_row: cutlass.Constexpr[Int32]+ page_entry_per_thread: Int32+ async_copy_elems: Int32++ gmem_tiled_copy_KV: cute.TiledCopy+ gmem_thr_copy_KV: cute.TiledCopy+ tPrPage: cute.Tensor+ tPrPageOffset: cute.Tensor+ tKpK: cute.Tensor+ tVpV: cute.Tensor++ @staticmethod+ def create(+ mPageTable: cute.Tensor,+ mK_paged: cute.Tensor,+ mV_paged: cute.Tensor,+ page_size_divmod: FastDivmod,+ bidb: Int32,+ bidh: Int32,+ thread_idx: Int32,+ seqlen_k: Int32,+ leftpad_k: Int32,+ n_block_size: cutlass.Constexpr[Int32],+ head_dim_padded: cutlass.Constexpr[Int32],+ head_dim_v_padded: cutlass.Constexpr[Int32],+ num_threads: cutlass.Constexpr[Int32],+ dtype: Type[cutlass.Numeric],+ ):+ universal_copy_bits = 128+ gmem_threads_per_row = 8 # 8 threads loading 128 bits = 128 bytes = 1 cache line+ async_copy_elems = universal_copy_bits // dtype.width+ atom_async_copy = cute.make_copy_atom(+ cpasync.CopyG2SOp(cache_mode=cpasync.LoadCacheMode.GLOBAL),+ dtype,+ num_bits_per_copy=universal_copy_bits,+ )+ thr_layout = cute.make_ordered_layout(+ (num_threads // gmem_threads_per_row, gmem_threads_per_row),+ order=(1, 0),+ )+ val_layout = cute.make_layout((1, async_copy_elems))+ gmem_tiled_copy_KV = cute.make_tiled_copy_tv(atom_async_copy, thr_layout, val_layout)+ gmem_thr_copy_KV = gmem_tiled_copy_KV.get_slice(thread_idx)+ page_entry_per_thread = n_block_size * gmem_threads_per_row // num_threads++ tPrPage = cute.make_rmem_tensor((page_entry_per_thread,), Int32)+ tPrPageOffset = cute.make_rmem_tensor((page_entry_per_thread,), Int32)++ mPageTable = mPageTable[bidb, None]+ mK_paged = mK_paged[None, None, bidh, None]+ mV_paged = mV_paged[None, None, bidh, None]++ cK = cute.make_identity_tensor((n_block_size, head_dim_padded))+ tKcK = gmem_thr_copy_KV.partition_S(cK)+ tKpK = utils.predicate_k(tKcK, limit=mK_paged.shape[1])++ if const_expr(head_dim_padded == head_dim_v_padded):+ tVpV = tKpK+ else:+ cV = cute.make_identity_tensor((n_block_size, head_dim_v_padded))+ tVcV = gmem_thr_copy_KV.partition_S(cV)+ tVpV = utils.predicate_k(tVcV, limit=mV_paged.shape[0])++ return PagedKVManager(+ mPageTable,+ mK_paged,+ mV_paged,+ thread_idx,+ page_size_divmod,+ seqlen_k,+ leftpad_k,+ n_block_size,+ num_threads,+ head_dim_padded,+ head_dim_v_padded,+ gmem_threads_per_row,+ page_entry_per_thread,+ async_copy_elems,+ gmem_tiled_copy_KV,+ gmem_thr_copy_KV,+ tPrPage,+ tPrPageOffset,+ tKpK,+ tVpV,+ )++ @cute.jit+ def load_page_table(self, n_block: Int32):+ for i in cutlass.range(self.page_entry_per_thread, unroll=1):+ row = (i * self.num_threads + self.thread_idx) // self.gmem_threads_per_row+ row_idx = n_block * self.n_block_size + row++ page_idx, page_offset = self.page_size_divmod.divmod(row_idx + self.leftpad_k)++ is_valid = (+ (i + 1) * self.num_threads <= self.n_block_size or row < self.n_block_size+ ) and row_idx < self.seqlen_k+ page = self.mPageTable[page_idx] if is_valid else 0++ self.tPrPage[i] = page+ self.tPrPageOffset[i] = page_offset++ @cute.jit+ def load_KV(self, n_block: Int32, sX: cute.Tensor, K_or_V: str):+ assert K_or_V in ("K", "V")++ # Finesse sX layout to be (M, N).+ sX_pi = cute.make_tensor(+ sX.iterator,+ cute.make_layout(+ (sX.shape[0][0], (sX.shape[0][1], sX.shape[2])),+ stride=(sX.stride[0][0], (sX.stride[0][1], sX.stride[2])),+ ),+ )++ if const_expr(K_or_V == "V"):+ # Need to transpose V+ sX_pi = cute.make_tensor(sX_pi.iterator, cute.select(sX_pi.layout, mode=[1, 0]))++ head_dim = self.head_dim_v_padded if const_expr(K_or_V == "V") else self.head_dim_padded+ cX = cute.make_identity_tensor((self.n_block_size, head_dim))+ tXsX = self.gmem_thr_copy_KV.partition_D(sX_pi)+ tXcX = self.gmem_thr_copy_KV.partition_S(cX)++ seqlenk_row_limit = self.seqlen_k - n_block * self.n_block_size if n_block >= 0 else 0+ for m in cutlass.range(cute.size(tXsX, mode=[1]), unroll=1):+ should_load = tXcX[0, m, 0][0] < seqlenk_row_limit++ page = self.tPrPage[m]+ page_offset = self.tPrPageOffset[m]+ mX_paged_cur = (+ self.mK_paged[page_offset, None, page]+ if const_expr(K_or_V == "K")+ else self.mV_paged[None, page_offset, page]+ )+ mX_paged_cur_copy = cute.tiled_divide(mX_paged_cur, (self.async_copy_elems,))++ if should_load:+ for k in cutlass.range(cute.size(tXsX, mode=[2]), unroll=1):+ ki = tXcX[0, 0, k][1] // self.async_copy_elems+ cute.copy(+ self.gmem_tiled_copy_KV,+ mX_paged_cur_copy[None, ki],+ tXsX[None, m, k],+ )+ elif const_expr(K_or_V == "V"):+ # Don't need to clear out the rest of the smem for K since we'll mask out the scores anyway.+ tXsX[None, m, None].fill(0)diff --git a/src/nvfp4_decode_kernel/_fa4/pipeline.py b/src/nvfp4_decode_kernel/_fa4/pipeline.pynew file mode 100644index 0000000..7ed7ab0--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/pipeline.py@@ -0,0 +1,365 @@+# Copyright (c) 2025, Tri Dao.++# import math+from typing import Optional+from dataclasses import dataclass++import cutlass+import cutlass.cute as cute+from cutlass import Boolean, Int32, const_expr+from cutlass.cutlass_dsl import if_generate+from cutlass.pipeline import PipelineAsync, PipelineState, Agent, CooperativeGroup+from cutlass.pipeline import PipelineUserType, PipelineOp+from cutlass.pipeline import PipelineTmaAsync as PipelineTmaAsyncOg+from cutlass.pipeline import PipelineTmaUmma as PipelineTmaUmmaOg+++# We deviate from cute-dsl implementation to use cute.arch.cluster_arrive_relaxed+def pipeline_init_wait(cta_layout_vmnk: Optional[cute.Layout] = None):+ """+ Fences the mbarrier init and syncs the threadblock or cluster+ """+ cute.arch.mbarrier_init_fence()++ if cta_layout_vmnk is None or cute.size(cta_layout_vmnk) == 1:+ # If not using clusters, sync the threadblock+ _sync(Agent.ThreadBlock)+ else:+ # If using clusters, sync the cluster+ _sync(Agent.ThreadBlockCluster)+++def _sync(group: Agent):+ """+ Syncs all threads within an agent.+ """+ if group is Agent.Thread:+ raise NotImplementedError("Error: Not supported.")+ elif group is Agent.ThreadBlock:+ cute.arch.sync_threads()+ elif group is Agent.ThreadBlockCluster:+ cute.arch.cluster_arrive_relaxed()+ cute.arch.cluster_wait()+ else:+ assert False, (+ "Error: No explicit sync instruction exists. Please use barriers (named / mbarrier) instead."+ )+++class PipelineStateSimple:+ """+ Pipeline state contains an index and phase bit corresponding to the current position in the circular buffer.+ Use a single Int32 to store both the index and phase bit, then we use divmod to get the+ index and phase. If stages is a power of 2, divmod turns into bit twiddling.+ """++ def __init__(self, stages: int, phase_index: Int32):+ # assert stages < 2**16+ # self._log_stages = int(math.log2(stages))+ # assert 1 << self._log_stages == stages, "Number of stages must be a power of 2."+ self._stages = stages+ self._phase_index = phase_index++ def clone(self) -> "PipelineStateSimple":+ return PipelineStateSimple(self.stages, self._phase_index)++ @property+ def stages(self) -> int:+ # return 1 << self._log_stages+ return self._stages++ @property+ def index(self) -> Int32:+ # return self._phase_index & 0xFFFF+ # return self._phase_index & ((1 << self._log_stages) - 1)+ if const_expr(self._stages == 1):+ return Int32(0)+ else:+ return self._phase_index % self._stages++ @property+ def phase(self) -> Int32:+ # return self._phase_index >> 16+ # PTX docs say that the phase parity needs to be 0 or 1, so by right we need to+ # take modulo 2. But in practice just passing the phase in without modulo works fine.+ # return (self._phase_index >> self._log_stages) % 2+ # return self._phase_index >> self._log_stages+ if const_expr(self._stages == 1):+ return self._phase_index+ else:+ return self._phase_index // self._stages++ def advance(self):+ if const_expr(self._stages == 1):+ self._phase_index ^= 1+ else:+ self._phase_index += 1++ # def then_body(phase_index):+ # # XOR the phase bit and set the index to 0+ # return (phase_index & 0xFFFF0000) ^ (1 << 16)++ # def else_body(phase_index):+ # return phase_index++ # self._phase_index = if_generate(+ # (self._phase_index & 0xFFFF) == self.stages,+ # then_body,+ # else_body,+ # [self._phase_index],+ # [Int32],+ # )++ def __extract_mlir_values__(self):+ phase_index = self._phase_index+ return [phase_index.ir_value()]++ def __new_from_mlir_values__(self, values):+ return PipelineStateSimple(self.stages, Int32(values[0]))+++def make_pipeline_state(type: PipelineUserType, stages: int):+ """+ Creates a pipeline state. Producers are assumed to start with an empty buffer and have a flipped phase bit of 1.+ """+ if type is PipelineUserType.Producer:+ # return PipelineStateSimple(stages, Int32(1 << 16))+ return PipelineStateSimple(stages, Int32(stages))+ elif type is PipelineUserType.Consumer:+ return PipelineStateSimple(stages, Int32(0))+ else:+ assert False, "Error: invalid PipelineUserType specified for make_pipeline_state."+++@dataclass(frozen=True)+class PipelineTmaAsync(PipelineTmaAsyncOg):+ """+ If size(ClusterShape) == 1, PipelineTmaAsync has all threads+ signaling the barrier during consumer_release. This causes a perf regression in FA3+ forward pass (especially hdim 128 causal). We instead implement a version of+ PipelineTmaAsync where only 1 out of 128 threads signals the barrier.++ Assumptions:+ (1) num_consumers % NumThreadsPerWarpGroup == 0+ (2) all 128 threads in the warp group are sync'ed right before calling consumer_release+ """++ @staticmethod+ def create(+ *,+ num_stages: int,+ producer_group: CooperativeGroup,+ consumer_group: CooperativeGroup,+ tx_count: int,+ barrier_storage: cute.Pointer = None,+ cta_layout_vmnk: Optional[cute.Layout] = None,+ tidx: Optional[Int32] = None,+ mcast_mode_mn: tuple[int, int] = (1, 1),+ init_wait: cutlass.Constexpr[bool] = True,+ ):+ """+ This helper function computes any necessary attributes and returns an instance of PipelineTmaAsync.+ :param barrier_storage: Pointer to the smem address for this pipeline's mbarriers+ :type barrier_storage: cute.Pointer+ :param num_stages: Number of buffer stages for this pipeline+ :type num_stages: Int32+ :param producer_group: `CooperativeGroup` for the producer agent+ :type producer_group: CooperativeGroup+ :param consumer_group: `CooperativeGroup` for the consumer agent+ :type consumer_group: CooperativeGroup+ :param tx_count: Number of bytes expected to be written to the transaction barrier for one stage+ :type tx_count: int+ :param cta_layout_vmnk: Layout of the cluster shape+ :type cta_layout_vmnk: cute.Layout | None+ :param tidx: thread index to consumer async threads+ :type tidx: Int32 | None+ :param mcast_mode_mn: Tuple of two integers, specifying whether mcast is enabled for the m and n modes. At least one of the two integers must be 1.+ :type mcast_mode_mn: tuple[int, int]+ """+ if not isinstance(barrier_storage, cute.Pointer):+ raise ValueError(+ f"Expected barrier_storage to be a cute.Pointer, but got {type(barrier_storage)}"+ )++ producer_type = PipelineOp.TmaLoad+ consumer_type = PipelineOp.AsyncThread++ producer = (producer_type, producer_group)+ consumer = (consumer_type, consumer_group)++ sync_object_full = PipelineAsync._make_sync_object(+ barrier_storage.align(min_align=8), num_stages, producer, tx_count+ )+ sync_object_empty = PipelineAsync._make_sync_object(+ barrier_storage.align(min_align=8) + num_stages, num_stages, consumer+ )+ if tidx is None:+ tidx, _, _ = cute.arch.thread_idx()+ if cta_layout_vmnk is None:+ cta_layout_vmnk = cute.make_layout((1, 1, 1, 1))+ if const_expr(cta_layout_vmnk is None or cute.size(cta_layout_vmnk) == 1):+ dst_rank = None+ is_signalling_thread = tidx % 128 == 0+ else:+ (+ dst_rank,+ is_signalling_thread,+ ) = PipelineTmaAsync.init_empty_barrier_arrive_signal(+ cta_layout_vmnk, tidx, mcast_mode_mn+ )++ producer_mask = None++ if const_expr(init_wait):+ pipeline_init_wait()++ return PipelineTmaAsync(+ sync_object_full,+ sync_object_empty,+ num_stages,+ producer_mask,+ dst_rank,+ is_signalling_thread,+ )++ def producer_acquire(+ self,+ state: PipelineState,+ try_acquire_token: Optional[Boolean] = None,+ extra_tx_count: int = 0,+ ):+ """+ TMA producer commit conditionally waits on buffer empty and sets the transaction barrier for leader threadblocks.+ """+ if_generate(+ try_acquire_token is None or try_acquire_token == 0,+ lambda: self.sync_object_empty.wait(state.index, state.phase),+ )+ if const_expr(extra_tx_count == 0):+ self.sync_object_full.arrive(state.index, self.producer_mask)+ else:+ tx_count = self.sync_object_full.tx_count + extra_tx_count+ self.sync_object_full.arrive_and_expect_tx(state.index, tx_count)++ def consumer_release(self, state: PipelineState):+ """+ TMA consumer release conditionally signals the empty buffer to the producer.+ """+ # Only 1 thread per warp group signals the empty buffer.+ if self.consumer_mask is None: # No cluster, 1 thread per warp group to signal+ if_generate(+ cute.arch.thread_idx()[0] % 128 == 0,+ lambda: self.sync_object_empty.arrive(state.index, self.consumer_mask),+ )+ else:+ if_generate(+ self.is_signalling_thread,+ lambda: self.sync_object_empty.arrive(state.index, self.consumer_mask),+ )+++@dataclass(frozen=True)+class PipelineTmaUmma(PipelineTmaUmmaOg):+ @staticmethod+ def create(+ *,+ num_stages: int,+ producer_group: CooperativeGroup,+ consumer_group: CooperativeGroup,+ tx_count: int,+ barrier_storage: cute.Pointer = None,+ cta_layout_vmnk: Optional[cute.Layout] = None,+ mcast_mode_mn: tuple[int, int] = (1, 1),+ init_wait: cutlass.Constexpr[bool] = True,+ ):+ """+ This helper function computes any necessary attributes and returns an instance of PipelineTmaUmma.+ :param barrier_storage: Pointer to the smem address for this pipeline's mbarriers+ :type barrier_storage: cute.Pointer+ :param num_stages: Number of buffer stages for this pipeline+ :type num_stages: Int32+ :param producer_group: `CooperativeGroup` for the producer agent+ :type producer_group: CooperativeGroup+ :param consumer_group: `CooperativeGroup` for the consumer agent+ :type consumer_group: CooperativeGroup+ :param tx_count: Number of bytes expected to be written to the transaction barrier for one stage+ :type tx_count: int+ :param cta_layout_vmnk: Layout of the cluster shape+ :type cta_layout_vmnk: cute.Layout | None+ :param mcast_mode_mn: Tuple of two integers, specifying whether mcast is enabled for the m and n modes. At least one of the two integers must be 1.+ :type mcast_mode_mn: tuple[int, int]+ """+ if not isinstance(barrier_storage, cute.Pointer):+ raise ValueError(+ f"Expected barrier_storage to be a cute.Pointer, but got {type(barrier_storage)}"+ )++ producer_type = PipelineOp.TmaLoad+ consumer_type = PipelineOp.TCGen05Mma++ producer = (producer_type, producer_group)+ consumer = (consumer_type, consumer_group)++ sync_object_full = PipelineAsync._make_sync_object(+ barrier_storage.align(min_align=8), num_stages, producer, tx_count+ )+ sync_object_empty = PipelineAsync._make_sync_object(+ barrier_storage.align(min_align=8) + num_stages, num_stages, consumer+ )++ if cta_layout_vmnk is None or cute.size(cta_layout_vmnk) == 1:+ # No mcast mask if not using clusters+ producer_mask = None+ # All threadblocks are leaders if not using clusters+ is_leader_cta = True+ else:+ producer_mask = PipelineTmaUmma._compute_mcast_arrival_mask(+ cta_layout_vmnk, mcast_mode_mn+ )+ is_leader_cta = PipelineTmaUmma._compute_is_leader_cta(cta_layout_vmnk)++ cta_group = (+ cute.nvgpu.tcgen05.CtaGroup.ONE+ if cta_layout_vmnk is None or cute.size(cta_layout_vmnk, mode=[0]) == 1+ else cute.nvgpu.tcgen05.CtaGroup.TWO+ )++ consumer_mask = producer_mask++ if const_expr(init_wait):+ pipeline_init_wait(cta_layout_vmnk)++ return PipelineTmaUmma(+ sync_object_full,+ sync_object_empty,+ num_stages,+ producer_mask,+ consumer_mask,+ is_leader_cta,+ cta_group,+ )++ def producer_acquire(+ self,+ state: PipelineState,+ try_acquire_token: Optional[Boolean] = None,+ extra_tx_count: int = 0,+ ):+ """+ TMA producer commit conditionally waits on buffer empty and sets the transaction barrier for leader threadblocks.+ """+ if_generate(+ try_acquire_token is None or try_acquire_token == 0,+ lambda: self.sync_object_empty.wait(state.index, state.phase),+ )+ if const_expr(extra_tx_count == 0):+ if_generate(+ self.is_leader_cta,+ lambda: self.sync_object_full.arrive(state.index, self.producer_mask),+ )+ else:+ tx_count = self.sync_object_full.tx_count + extra_tx_count+ if_generate(+ self.is_leader_cta,+ lambda: self.sync_object_full.arrive_and_expect_tx(state.index, tx_count),+ )diff --git a/src/nvfp4_decode_kernel/_fa4/seqlen_info.py b/src/nvfp4_decode_kernel/_fa4/seqlen_info.pynew file mode 100644index 0000000..49e9ea1--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/seqlen_info.py@@ -0,0 +1,123 @@+from typing import Optional+from dataclasses import dataclass++import cutlass+import cutlass.cute as cute+from cutlass import Int32, const_expr++"""+This consolidates all the info related to sequence length. This is so that we can do all+the gmem reads once at the beginning of each tile, rather than having to repeat these reads+to compute various things like n_block_min, n_block_max, etc.+"""+++@dataclass(frozen=True)+class SeqlenInfo:+ offset: cutlass.Int32+ seqlen: cutlass.Int32++ @staticmethod+ def create(+ batch_idx: cutlass.Int32,+ seqlen_static: cutlass.Int32,+ cu_seqlens: Optional[cute.Tensor] = None,+ seqused: Optional[cute.Tensor] = None,+ ):+ offset = 0 if const_expr(cu_seqlens is None) else cu_seqlens[batch_idx]+ if const_expr(seqused is not None):+ seqlen = seqused[batch_idx]+ elif const_expr(cu_seqlens is not None):+ seqlen = cu_seqlens[batch_idx + 1] - cu_seqlens[batch_idx]+ else:+ seqlen = seqlen_static+ return SeqlenInfo(offset, seqlen)++ def offset_batch(self, tensor: cute.Tensor, batch_idx: Int32, dim: int) -> cute.Tensor:+ """Offset a fixed-batch or varlen tensor along its batch/sequence axis."""+ idx = (None,) * dim + (batch_idx,) + (None,) * (cute.rank(tensor) - 1 - dim)+ return tensor[idx]+++@dataclass(frozen=True)+class SeqlenInfoQK:+ offset_q: cutlass.Int32+ offset_k: cutlass.Int32+ seqlen_q: cutlass.Int32+ seqlen_k: cutlass.Int32+ has_cu_seqlens_q: cutlass.Constexpr[bool]+ has_cu_seqlens_k: cutlass.Constexpr[bool]++ @staticmethod+ def create(+ batch_idx: cutlass.Int32,+ seqlen_q_static: cutlass.Int32,+ seqlen_k_static: cutlass.Int32,+ mCuSeqlensQ: Optional[cute.Tensor] = None,+ mCuSeqlensK: Optional[cute.Tensor] = None,+ mSeqUsedQ: Optional[cute.Tensor] = None,+ mSeqUsedK: Optional[cute.Tensor] = None,+ ):+ offset_q = 0 if const_expr(mCuSeqlensQ is None) else mCuSeqlensQ[batch_idx]+ offset_k = 0 if const_expr(mCuSeqlensK is None) else mCuSeqlensK[batch_idx]+ if const_expr(mSeqUsedQ is not None):+ seqlen_q = mSeqUsedQ[batch_idx]+ else:+ seqlen_q = (+ seqlen_q_static+ if const_expr(mCuSeqlensQ is None)+ else mCuSeqlensQ[batch_idx + 1] - offset_q+ )+ if const_expr(mSeqUsedK is not None):+ seqlen_k = mSeqUsedK[batch_idx]+ else:+ seqlen_k = (+ seqlen_k_static+ if const_expr(mCuSeqlensK is None)+ else mCuSeqlensK[batch_idx + 1] - offset_k+ )+ has_cu_seqlens_q: int = mCuSeqlensQ is not None+ has_cu_seqlens_k: int = mCuSeqlensK is not None+ return SeqlenInfoQK(+ offset_q, offset_k, seqlen_q, seqlen_k, has_cu_seqlens_q, has_cu_seqlens_k+ )++ def offset_batch_Q(self, mQ: cute.Tensor, batch_idx: Int32, dim: int) -> cute.Tensor:+ """Seqlen must be the first dimension of mQ"""+ if const_expr(not self.has_cu_seqlens_q):+ idx = (None,) * dim + (batch_idx,) + (None,) * (cute.rank(mQ) - 1 - dim)+ return mQ[idx]+ else:+ offset = (+ self.offset_q if const_expr(cute.rank(mQ.shape[0]) == 1) else (0, self.offset_q)+ )+ idx = (offset,) + (0,) * (cute.rank(mQ) - 1)+ return cute.domain_offset(idx, mQ)++ def offset_batch_K(self, mK: cute.Tensor, batch_idx: Int32, dim: int) -> cute.Tensor:+ """Seqlen must be the first dimension of mK"""+ if const_expr(not self.has_cu_seqlens_k):+ idx = (None,) * dim + (batch_idx,) + (None,) * (cute.rank(mK) - 1 - dim)+ return mK[idx]+ else:+ idx = (self.offset_k,) + (0,) * (cute.rank(mK) - 1)+ return cute.domain_offset(idx, mK)++ def offset_batch_O_via_indices(+ self,+ mO: cute.Tensor,+ batch_idx: Int32,+ mOutIndices: cute.Tensor,+ dim: int,+ ) -> cute.Tensor:+ """Substitute for `offset_batch_Q` when output rows are indirect.++ Slice mO along `dim` at row+ `mOutIndices[batch_idx]`, producing the same shape+ `offset_batch_Q(..., dim)` would on the non-varlen path.+ mOutIndices[batch_idx] is the per-batch destination row in the flat+ global output (NOT a CDF — arbitrary positions per decode slot).+ """+ offset = mOutIndices[batch_idx]+ idx = (None,) * dim + (offset,) + (None,) * (cute.rank(mO) - 1 - dim)+ return mO[idx]diff --git a/src/nvfp4_decode_kernel/_fa4/softmax.py b/src/nvfp4_decode_kernel/_fa4/softmax.pynew file mode 100644index 0000000..24cd62d--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/softmax.py@@ -0,0 +1,764 @@+# Copyright (c) 2025, Tri Dao.++from typing import Optional+import math+import operator+import struct+from typing import Tuple+from dataclasses import dataclass++import cutlass+import cutlass.cute as cute+from cutlass import Float32+from cutlass._mlir.dialects import llvm+from cutlass.cutlass_dsl import T++from . import utils as utils+from .cute_dsl_utils import ParamsBase+from cutlass import const_expr++from .._block_scale import _fma, _widen_e2m1_byte, power_of_two, inverse_power_of_two+from ..p_scale import P_SMALL_GROUP_AMAX+from .blackwell_helpers import (+ packed_float_to_e2m1x2,+ packed_float_to_ue4m3,+)+++def _round_probability_scale(value: Float32) -> Float32:+ rounded = cute.make_rmem_tensor(+ cute.make_layout(4), cutlass.Float8E4M3FN+ )+ cute.recast_tensor(rounded, cutlass.Int32)[0] = packed_float_to_ue4m3(+ value, Float32(0.0), Float32(0.0), Float32(0.0)+ )+ return Float32(rounded.load().to(Float32)[0])+++def _round_probability_code(value: Float32) -> Float32:+ packed = packed_float_to_e2m1x2(Float32(0.0), value)+ widened = cute.make_rmem_tensor(cute.make_layout(2), cutlass.Float16)+ cute.recast_tensor(widened, cutlass.Int32)[0] = _widen_e2m1_byte(+ packed, 0+ )+ return Float32(widened[0])+++def _mxfp4_probability_exponent(probability: Float32, exponent: cutlass.Int32) -> cutlass.Int32:+ """One P value per lane; the full warp shares one E8M0 scale.++ Compare round-up with the adjacent smaller scale. The signed error sum+ penalizes lost or added group mass without changing the softmax denominator.+ Strict comparison retains round-up on ties, including fully masked groups.+ """+ smaller = cutlass.Int32(cutlass.select_(exponent > -127, exponent - 1, exponent))+ best_cost = Float32(0.0)+ rival_cost = Float32(0.0)+ for candidate in cutlass.range_constexpr(2):+ chosen = exponent if cutlass.const_expr(candidate == 0) else smaller+ code = _round_probability_code(probability * inverse_power_of_two(chosen))+ difference = _fma(code, power_of_two(chosen), -probability)+ squared = difference * difference+ mass = difference+ for step in cutlass.range_constexpr(5):+ squared += cute.arch.shuffle_sync_bfly(squared, offset=1 << step)+ mass += cute.arch.shuffle_sync_bfly(mass, offset=1 << step)+ cost = _fma(mass * Float32(1.0 / 32.0), mass, squared)+ if cutlass.const_expr(candidate == 0):+ best_cost = cost+ else:+ rival_cost = cost+ return cutlass.Int32(cutlass.select_(rival_cost < best_cost, smaller, exponent))+++def _ptx_f32(value: float) -> str:+ return "0f" + struct.pack(">f", value).hex()+++_AMAX2_OR_AMAX6_PTX = (+ "{\n\t"+ ".reg .pred low;\n\t"+ f"setp.le.f32 low, $1, {_ptx_f32(P_SMALL_GROUP_AMAX)};\n\t"+ f"mul.rn.f32 $0, $1, {_ptx_f32(1.0 / 6.0)};\n\t"+ f"@low mul.rn.f32 $0, $1, {_ptx_f32(1.0 / 2.0)};\n\t"+ "}\n"+)+++def _amax2_or_amax6_probability_scale(+ maximum: Float32, *, loc=None, ip=None+) -> Float32:+ """Return amax/2 when ``maximum <= P_SMALL_GROUP_AMAX``, else amax/6."""+ # Keep the predicate and alternate multiply inside one inline block so the+ # compiler only has to keep the returned scale live. Materializing the+ # selected reciprocal in DSL IR crossed a register-allocation boundary in+ # the 128K kernel and cost more than the arithmetic itself.+ scale = llvm.inline_asm(+ T.f32(),+ [Float32(maximum).ir_value(loc=loc, ip=ip)],+ _AMAX2_OR_AMAX6_PTX,+ "=f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ loc=loc,+ ip=ip,+ )+ return Float32(scale)+++@dataclass+class Softmax(ParamsBase):+ scale_log2: Float32+ num_rows: cutlass.Constexpr[int]+ row_max: cute.Tensor+ row_sum: cute.Tensor+ arch: cutlass.Constexpr[int] = 80+ softmax_scale: Float32 | None = None++ @staticmethod+ def create(+ scale_log2: Float32,+ num_rows: cutlass.Constexpr[int],+ arch: cutlass.Constexpr[int] = 80,+ softmax_scale: Float32 | None = None,+ ):+ row_max = cute.make_rmem_tensor(num_rows, Float32)+ row_sum = cute.make_rmem_tensor(num_rows, Float32)+ return Softmax(scale_log2, num_rows, row_max, row_sum, arch, softmax_scale)++ def reset(self) -> None:+ self.row_max.fill(-Float32.inf)+ self.row_sum.fill(0.0)++ def _compute_row_max(+ self, acc_S_row: cute.TensorSSA, init_val: float | Float32 | None = None+ ) -> Float32:+ return utils.fmax_reduce(acc_S_row, init_val, arch=self.arch)++ def _compute_row_sum(+ self, acc_S_row_exp: cute.TensorSSA, init_val: float | Float32 | None = None+ ) -> Float32:+ return utils.fadd_reduce(acc_S_row_exp, init_val, arch=self.arch)++ @cute.jit+ def online_softmax(+ self,+ acc_S: cute.Tensor,+ is_first: cutlass.Constexpr[bool] = False,+ check_inf: cutlass.Constexpr[bool] = True,+ ) -> cute.Tensor:+ """Apply online softmax and return the row_scale to rescale O.++ :param acc_S: acc_S tensor+ :type acc_S: cute.Tensor+ :param is_first: is first n_block+ :type is_first: cutlass.Constexpr+ """+ # Change acc_S to M,N layout view.+ acc_S_mn = utils.make_acc_tensor_mn_view(acc_S)+ row_scale = cute.make_fragment_like(self.row_max, Float32)++ row_max = self.row_max+ row_sum = self.row_sum+ scale_log2 = self.scale_log2+ arch = self.arch++ # Each iteration processes one row of acc_S+ for r in cutlass.range(cute.size(row_max), unroll_full=True):+ acc_S_row = acc_S_mn[r, None].load() # (n_block_size)++ row_max_cur = utils.fmax_reduce(+ acc_S_row,+ init_val=row_max[r] if cutlass.const_expr(not is_first) else None,+ arch=arch,+ )++ row_max_cur = utils.warp_reduce(row_max_cur, cute.arch.fmax, width=4)+ if cutlass.const_expr(check_inf):+ row_max_cur = 0.0 if row_max_cur == -Float32.inf else row_max_cur++ if cutlass.const_expr(is_first):+ row_max_cur_scaled = row_max_cur * scale_log2+ acc_S_row_exp = utils.exp2f(acc_S_row * scale_log2 - row_max_cur_scaled)++ acc_S_row_sum = utils.fadd_reduce(acc_S_row_exp, init_val=None, arch=arch)+ row_scale[r] = 1.0+ else:+ row_max_prev = row_max[r]+ row_max_cur_scaled = row_max_cur * scale_log2+ acc_S_row_exp = utils.exp2f(acc_S_row * scale_log2 - row_max_cur_scaled)+ # row_scale[r] = utils.exp2f(row_max_prev * self.scale_log2 - row_max_cur_scaled)+ row_scale[r] = utils.exp2f((row_max_prev - row_max_cur) * scale_log2)++ acc_S_row_sum = utils.fadd_reduce(+ acc_S_row_exp, init_val=row_sum[r] * row_scale[r], arch=arch+ )++ row_max[r] = row_max_cur+ row_sum[r] = acc_S_row_sum+ acc_S_mn[r, None].store(acc_S_row_exp)++ return row_scale++ @cute.jit+ def finalize(+ self, final_scale: Float32 = 1.0, sink_val: Float32 | cute.Tensor | None = None+ ) -> cute.Tensor:+ """Finalize the online softmax by computing the scale and logsumexp."""+ if cutlass.const_expr(sink_val is not None and isinstance(sink_val, cute.Tensor)):+ assert cute.size(sink_val) == cute.size(self.row_sum)+ row_sum = self.row_sum+ row_max = self.row_max+ scale_log2 = self.scale_log2++ # quad reduction for row_sum as we didn't do it during each iteration of online softmax+ row_sum.store(utils.warp_reduce(row_sum.load(), operator.add, width=4))+ row_scale = cute.make_fragment_like(row_max, Float32)++ for r in cutlass.range(cute.size(row_sum), unroll_full=True):+ if cutlass.const_expr(sink_val is not None):+ sink_val_cur = sink_val if not isinstance(sink_val, cute.Tensor) else sink_val[r]+ LOG2_E = math.log2(math.e)+ row_sum[r] += utils.exp2f(sink_val_cur * LOG2_E - row_max[r] * scale_log2)++ # if row_sum is zero or nan, set acc_O_mn_row to 1.0+ acc_O_mn_row_is_zero_or_nan = row_sum[r] == 0.0 or row_sum[r] != row_sum[r]+ row_scale[r] = (+ cute.arch.rcp_approx(row_sum[r] if not acc_O_mn_row_is_zero_or_nan else 1.0)+ ) * final_scale+ row_sum_cur = row_sum[r]+ LN2 = math.log(2.0)+ row_sum[r] = (+ (row_max[r] * scale_log2 + utils.log2f(row_sum_cur)) * LN2+ if not acc_O_mn_row_is_zero_or_nan+ else -Float32.inf+ )+ return row_scale++ @cute.jit+ def rescale_O(self, acc_O: cute.Tensor, row_scale: cute.Tensor) -> None:+ """Scale each row of acc_O by the given scale tensor.+ :param acc_O: input tensor+ :type acc_O: cute.Tensor+ :param row_scale: row_scale tensor+ :type row_scale: cute.Tensor+ """+ acc_O_mn = utils.make_acc_tensor_mn_view(acc_O)+ assert cute.size(row_scale) == cute.size(acc_O_mn, mode=[0])+ for r in cutlass.range(cute.size(row_scale), unroll_full=True):+ acc_O_mn[r, None].store(acc_O_mn[r, None].load() * row_scale[r])+++@dataclass+class SoftmaxSm100(Softmax):+ rescale_threshold: cutlass.Constexpr[float] = 0.0++ fp8_scalexfp4_scale_log2: cutlass.Constexpr[float] = -11.392317422778762 # log2f(fp8_scalexfp4_scale=1.0 / (448 * 6))+ fp4_scale_log2: cutlass.Constexpr[float] = -2.584962500721156 # log2f(fp4_scale=1 / 6)+ quant_pv: cutlass.Constexpr[bool] = False+ compute_sp1: cutlass.Constexpr[bool] = False+ p_scale_divisor: cutlass.Constexpr[float] = 6.0+ p_scale_grid_search: cutlass.Constexpr[bool] = False+ p_scale_amax2_or_amax6: cutlass.Constexpr[bool] = False+ # Optional Kahan-compensated FP32 accumulation for row_sum.+ # Compensation register tracks lost low-order bits across the+ # online-softmax rescale + add steps. A single Float32 per softmax+ # thread; lives in registers, with no TMEM or SMEM cost.+ kahan: cutlass.Constexpr[bool] = False+ row_sum_comp: Optional[cute.Tensor] = None++ @staticmethod+ def create(+ scale_log2: Float32,+ rescale_threshold: cutlass.Constexpr[float] = 0.0,+ softmax_scale: Float32 | None = None,+ quant_pv: cutlass.Constexpr[bool] = False,+ compute_sp1: cutlass.Constexpr[bool] = False,+ p_scale_divisor: cutlass.Constexpr[float] = 6.0,+ p_scale_grid_search: cutlass.Constexpr[bool] = False,+ p_scale_amax2_or_amax6: cutlass.Constexpr[bool] = False,+ kahan: cutlass.Constexpr[bool] = False,+ ):+ num_rows = 1+ arch = 100+ row_max = cute.make_rmem_tensor(num_rows, Float32)+ row_sum = cute.make_rmem_tensor(num_rows, Float32)+ row_sum_comp = cute.make_rmem_tensor(num_rows, Float32) if cutlass.const_expr(kahan) else None++ return SoftmaxSm100(+ scale_log2,+ num_rows,+ row_max,+ row_sum,+ arch,+ softmax_scale,+ rescale_threshold=rescale_threshold,+ quant_pv=quant_pv,+ compute_sp1=compute_sp1,+ p_scale_divisor=p_scale_divisor,+ p_scale_grid_search=p_scale_grid_search,+ p_scale_amax2_or_amax6=p_scale_amax2_or_amax6,+ kahan=kahan,+ row_sum_comp=row_sum_comp,+ )++ def reset(self) -> None:+ self.row_max.fill(-Float32.inf)+ self.row_sum.fill(0.0)+ if cutlass.const_expr(self.kahan):+ self.row_sum_comp.fill(0.0)++ @cute.jit+ def update_row_max(self, acc_S_row: cute.TensorSSA, is_first: int) -> Tuple[Float32, Float32]:+ if cutlass.const_expr(is_first):+ row_max_new = self._compute_row_max(acc_S_row)+ row_max_safe = row_max_new if row_max_new != -cutlass.Float32.inf else 0.0+ acc_scale = 0.0+ else:+ row_max_old = self.row_max[0]+ row_max_new = self._compute_row_max(acc_S_row, init_val=row_max_old)+ row_max_safe = row_max_new if row_max_new != -cutlass.Float32.inf else 0.0+ acc_scale_ = (row_max_old - row_max_safe) * self.scale_log2+ acc_scale = utils.exp2f(acc_scale_)+ if cutlass.const_expr(self.rescale_threshold > 0.0):+ if acc_scale_ >= -self.rescale_threshold:+ row_max_new = row_max_old+ row_max_safe = row_max_old+ acc_scale = 1.0+ self.row_max[0] = row_max_new+ return row_max_safe, acc_scale++ @cute.jit+ def compute_group_max(self, acc_S_row: cute.Tensor, sf_size: cutlass.Constexpr[int] = 16) -> cute.Tensor:+ """Per-group max for FP4 P quantization.++ Two bugs in the original code:+ 1. No cross-thread reduce. The tcgen05 t2r load distributes a K group of 16+ elements across multiple lanes; each thread sees only its slice. The+ per-thread fmax_reduce returns the local max, not the per-group max,+ so SF_p stored to TMEM is wrong for most positions.+ 2. Returned plain max instead of max/6, so scale_groupwise produced+ P/max ∈ [0, 1]. FP4 quantization in [0, 1] only ever uses nibbles+ {0, 0.5, 1.0} — 3 levels of precision per element, vs the full 8+ ({0, 0.5, 1, 1.5, 2, 3, 4, 6}) when scaling to [0, 6].++ Width-32 warp_reduce is empirically the best (cos jumps from 0.62 → 0.90).+ The exact lane sharing depends on the tcgen05 partition; over-reducing to+ the full warp is conservative (every group sees the warp-wide max) and+ loses some per-group adaptivity but correctness dominates.+ """+ acc_S_row_frag = cute.logical_divide(acc_S_row, cute.make_layout(sf_size))+ num_frags = cute.size(acc_S_row_frag, mode=[1])+ acc_S_row_group_max = cute.make_rmem_tensor(cute.make_layout(num_frags), Float32)+ for i in cutlass.range_constexpr(num_frags):+ # Each thread holds the entire K group (16 elements) for one M row of+ # the t2r tile, so per-thread fmax_reduce IS the per-group max — no+ # cross-lane reduce needed.+ values = acc_S_row_frag[None, i].load()+ maximum = self._compute_row_max(values)+ if cutlass.const_expr(self.p_scale_amax2_or_amax6):+ acc_S_row_group_max[i] = (+ _amax2_or_amax6_probability_scale(maximum)+ )+ elif cutlass.const_expr(self.p_scale_grid_search):+ best_scale = maximum * Float32(1.0 / 6.0)+ best_stored = _round_probability_scale(best_scale)+ best_inverse = Float32(1.0) / cute.arch.fmax(+ best_scale, 1e-20+ )+ best_error = Float32(0.0)+ for element in cutlass.range_constexpr(sf_size):+ code = _round_probability_code(+ Float32(values[element]) * best_inverse+ )+ difference = _fma(+ code, best_stored, -Float32(values[element])+ )+ best_error = _fma(difference, difference, best_error)+ rival_divisors = (4.0, 3.0, 2.0)+ for candidate in cutlass.range_constexpr(+ len(rival_divisors)+ ):+ divisor = rival_divisors[candidate]+ rival_scale = maximum * Float32(1.0 / divisor)+ rival_stored = _round_probability_scale(rival_scale)+ rival_inverse = Float32(1.0) / cute.arch.fmax(+ rival_scale, 1e-20+ )+ rival_error = Float32(0.0)+ for element in cutlass.range_constexpr(sf_size):+ code = _round_probability_code(+ Float32(values[element]) * rival_inverse+ )+ difference = _fma(+ code, rival_stored, -Float32(values[element])+ )+ rival_error = _fma(+ difference, difference, rival_error+ )+ better = rival_error < best_error+ best_scale = Float32(+ cutlass.select_(better, rival_scale, best_scale)+ )+ best_error = Float32(+ cutlass.select_(better, rival_error, best_error)+ )+ acc_S_row_group_max[i] = best_scale+ else:+ acc_S_row_group_max[i] = (+ maximum * Float32(1.0 / self.p_scale_divisor)+ )+ return acc_S_row_group_max++ @cute.jit+ def scale_groupwise(self, acc_S_row: cute.Tensor, group_max: cute.Tensor, sf_size: cutlass.Constexpr[int] = 16):+ """Normalize P by per-group max before FP4 quantization.++ After this: P_fp4 * SFP ≈ (P/gmax) * gmax = P.+ """+ acc_S_row_frag = cute.logical_divide(acc_S_row, cute.make_layout(sf_size))+ for g in cutlass.range_constexpr(cute.size(group_max)):+ denominator = cute.arch.fmax(group_max[g], 1e-20)+ # The alternate transposed-S implementation already uses the+ # hardware reciprocal here. E2M1 has only two significant bits,+ # so its approximation does not buy representable precision, and+ # using it for the threshold candidate offsets the one predicate+ # that chooses /2 versus /6 in each hot P group.+ inv_gmax = (+ cute.arch.rcp_approx(denominator)+ if cutlass.const_expr(self.p_scale_amax2_or_amax6)+ else Float32(1.0) / denominator+ )+ for j in cutlass.range(0, sf_size, 2, unroll_full=True):+ acc_S_row_frag[j, g], acc_S_row_frag[j + 1, g] = utils.mul_packed_f32x2(+ (acc_S_row_frag[j, g], acc_S_row_frag[j + 1, g]),+ (inv_gmax, inv_gmax),+ )++ def update_row_sum(+ self, acc_S_row_exp: cute.TensorSSA, row_scale: Float32, is_first: int = False+ ) -> None:+ if cutlass.const_expr(self.kahan):+ # Kahan-compensated row_sum update.+ # Captures lost bits across rescale + add. Equivalent to:+ # row_sum_new = row_sum * row_scale + sum(acc_S_row_exp)+ # but tracks the rounding error in row_sum_comp so the+ # epilogue (`row_sum + row_sum_comp` before division into o)+ # recovers the lost low-order bits.+ #+ # is_first=True: row_sum = row_scale = 0; comp stays at 0.+ # is_first=False: standard Kahan around the scalar+vec sum:+ # contrib = sum(acc_S_row_exp)+ # init = row_sum * row_scale+ # comp = comp * row_scale # carry comp through rescale+ # y = contrib - comp+ # t = init + y+ # comp = (t - init) - y+ # row_sum = t+ contrib = self._compute_row_sum(acc_S_row_exp, init_val=None)+ if cutlass.const_expr(is_first):+ # First block: row_sum = 0, comp stays 0. Skip Kahan to+ # match the legacy is_first=True semantics exactly.+ self.row_sum[0] = contrib+ self.row_sum_comp[0] = Float32(0.0)+ else:+ init = self.row_sum[0] * row_scale+ comp_rescaled = self.row_sum_comp[0] * row_scale+ y = contrib - comp_rescaled+ t = init + y+ self.row_sum_comp[0] = (t - init) - y+ self.row_sum[0] = t+ else:+ init_val = self.row_sum[0] * row_scale if cutlass.const_expr(not is_first) else None+ # self.row_sum[0] = self._compute_row_sum(acc_S_row_exp, init_val=self.row_sum[0] * row_scale)+ self.row_sum[0] = self._compute_row_sum(acc_S_row_exp, init_val=init_val)+ # tmp = self._compute_row_sum(acc_S_row_exp)++ @cute.jit+ def update_row_sum_sage(+ self, acc_S_row_exp: cute.Tensor, acc_S_row_group_max_exp: cute.Tensor, row_scale: Float32, is_first: int = False+ ) -> None:+ init_val = self.row_sum[0] * row_scale if cutlass.const_expr(not is_first) else None+ vec_size = const_expr(cute.size(acc_S_row_exp) // cute.size(acc_S_row_group_max_exp))+ acc_S_row_exp_frag = cute.logical_divide(acc_S_row_exp, cute.make_layout(vec_size))+ acc_S_group_sum = cute.make_rmem_tensor(acc_S_row_group_max_exp.layout, dtype=Float32)++ num_groups = cute.size(acc_S_row_exp_frag, mode=[1])+ assert num_groups % 2 == 0+ for i in cutlass.range_constexpr(0, num_groups, 2):+ acc_S_group_sum[i], acc_S_group_sum[i + 1] = utils.mul_packed_f32x2(+ (self._compute_row_sum(acc_S_row_exp_frag[None, i].load()), self._compute_row_sum(acc_S_row_exp_frag[None, i + 1].load())),+ (acc_S_row_group_max_exp[i], acc_S_row_group_max_exp[i + 1]),+ )+ self.row_sum[0]= self._compute_row_sum(acc_S_group_sum.load(), init_val=init_val)++ @cute.jit+ def scale_subtract_rowmax(+ self,+ acc_S_row: cute.Tensor,+ row_max: Float32,+ acc_S_row_group_max: Optional[cute.Tensor] = None,+ ):+ assert cute.size(acc_S_row.shape) % 2 == 0, "acc_S_row must have an even number of elements"++ row_max_scaled = Float32(0.0)+ if const_expr(self.compute_sp1) and const_expr(self.quant_pv):+ row_max_scaled = row_max * self.scale_log2 + self.fp8_scalexfp4_scale_log2+ else:+ row_max_scaled = row_max * self.scale_log2++ if const_expr(acc_S_row_group_max is not None):+ if const_expr(self.compute_sp1) and const_expr(self.quant_pv):+ row_max_scaled += self.fp4_scale_log2 # / 6 to scale to fp4 max+ for i in cutlass.range(0, cute.size(acc_S_row_group_max.shape), 2, unroll_full=True):+ acc_S_row_group_max[i], acc_S_row_group_max[i + 1] = utils.fma_packed_f32x2(+ (acc_S_row_group_max[i], acc_S_row_group_max[i + 1]),+ (self.scale_log2, self.scale_log2),+ (-row_max_scaled, -row_max_scaled),+ )+ vec_size = cute.size(acc_S_row) // cute.size(acc_S_row_group_max)+ acc_S_row_frag = cute.logical_divide(acc_S_row, cute.make_layout(vec_size))+ for j in cutlass.range_constexpr(0, cute.size(acc_S_row_frag, mode=[1])):+ for i in cutlass.range(0, cute.size(acc_S_row_frag, mode=[0]), 2, unroll_full=True):+ bias = -row_max_scaled - acc_S_row_group_max[j]+ acc_S_row_frag[i, j], acc_S_row_frag[i + 1, j] = utils.fma_packed_f32x2(+ (acc_S_row_frag[i, j], acc_S_row_frag[i + 1, j]),+ (self.scale_log2, self.scale_log2),+ (bias, bias),+ )+ else:+ for i in cutlass.range(0, cute.size(acc_S_row.shape), 2, unroll_full=True):+ acc_S_row[i], acc_S_row[i + 1] = utils.fma_packed_f32x2(+ (acc_S_row[i], acc_S_row[i + 1]),+ (self.scale_log2, self.scale_log2),+ (-row_max_scaled, -row_max_scaled),+ )++ @cute.jit+ def apply_exp2_convert(+ self,+ acc_S_row: cute.Tensor,+ acc_S_row_converted: Optional[cute.Tensor] = None,+ e2e: cutlass.Constexpr[bool] = False,+ e2e_freq: cutlass.Constexpr[int] = 16,+ e2e_res: cutlass.Constexpr[int] = 4,+ e2e_frg_limit: cutlass.Constexpr[int] = 1,+ ):+ assert cute.size(acc_S_row.shape) % 2 == 0, "acc_S_row must have an even number of elements"+ frg_tile = min(32, cute.size(acc_S_row))+ assert frg_tile % 2 == 0+ frg_cnt = cute.size(acc_S_row) // frg_tile+ assert cute.size(acc_S_row) % frg_tile == 0+ acc_S_row_frg = cute.logical_divide(acc_S_row, cute.make_layout(frg_tile))+ for j in cutlass.range_constexpr(frg_cnt):+ for k in cutlass.range_constexpr(0, cute.size(acc_S_row_frg, mode=[0]), 2):+ # acc_S_row_frg[k, j] = utils.exp2f(acc_S_row_frg[k, j])+ # acc_S_row_frg[k + 1, j] = utils.exp2f(acc_S_row_frg[k + 1, j])+ if cutlass.const_expr(not e2e):+ acc_S_row_frg[k, j] = cute.arch.exp2(acc_S_row_frg[k, j])+ acc_S_row_frg[k + 1, j] = cute.arch.exp2(acc_S_row_frg[k + 1, j])+ else:+ if cutlass.const_expr(+ k % e2e_freq < e2e_freq - e2e_res or j >= frg_cnt - e2e_frg_limit+ ):+ acc_S_row_frg[k, j] = cute.arch.exp2(acc_S_row_frg[k, j])+ acc_S_row_frg[k + 1, j] = cute.arch.exp2(acc_S_row_frg[k + 1, j])+ else:+ # acc_S_row_frg[k, j], acc_S_row_frg[k + 1, j] = utils.e2e_asm2(acc_S_row_frg[k, j], acc_S_row_frg[k + 1, j])+ acc_S_row_frg[k, j], acc_S_row_frg[k + 1, j] = utils.ex2_emulation_2(+ acc_S_row_frg[k, j], acc_S_row_frg[k + 1, j]+ )+ if cutlass.const_expr(acc_S_row_converted is not None):+ acc_S_row_converted_frg = cute.logical_divide(+ acc_S_row_converted, cute.make_layout(frg_tile)+ )+ acc_S_row_converted_frg[None, j].store(+ acc_S_row_frg[None, j].load().to(acc_S_row_converted.element_type)+ )++ @cute.jit+ def apply_sage_sp1(self, P_row: cute.Tensor):+ row_max = self._compute_row_max(P_row.load())+ sp1 = row_max * self.sp1_scale+ inv_sp1 = 1.0 / sp1+ P_row.store(P_row.load() * inv_sp1)+ return sp1++ @cute.jit+ def scale_apply_exp2_convert(+ self,+ acc_S_row: cute.Tensor,+ row_max: Float32,+ acc_S_row_converted: cute.Tensor,+ ):+ assert cute.size(acc_S_row.shape) % 2 == 0, "acc_S_row must have an even number of elements"+ minus_row_max_scaled = -row_max * self.scale_log2+ for i in cutlass.range_constexpr(0, cute.size(acc_S_row.shape), 2):+ acc_S_row[i], acc_S_row[i + 1] = utils.fma_packed_f32x2(+ (acc_S_row[i], acc_S_row[i + 1]),+ (self.scale_log2, self.scale_log2),+ (minus_row_max_scaled, minus_row_max_scaled),+ )++ # for i in cutlass.range_constexpr(0, cute.size(acc_S_row.shape), 2):+ # acc_S_row[i], acc_S_row[i + 1] = utils.fma_packed_f32x2(+ # (acc_S_row[i], acc_S_row[i + 1]),+ # (self.scale_log2, self.scale_log2),+ # (minus_row_max_scaled, minus_row_max_scaled),+ # )+ # acc_S_row[i] = cute.arch.exp2(acc_S_row[i])+ # acc_S_row[i + 1] = cute.arch.exp2(acc_S_row[i + 1])++ frg_tile = 32+ assert frg_tile % 2 == 0+ frg_cnt = cute.size(acc_S_row) // frg_tile+ assert cute.size(acc_S_row) % frg_tile == 0+ acc_S_row_frg = cute.logical_divide(acc_S_row, cute.make_layout(frg_tile))+ acc_S_row_converted_frg = cute.logical_divide(+ acc_S_row_converted, cute.make_layout(frg_tile)+ )+ for j in cutlass.range_constexpr(frg_cnt):+ for k in cutlass.range_constexpr(0, cute.size(acc_S_row_frg, mode=[0]), 2):+ # acc_S_row_frg[k, j], acc_S_row_frg[k + 1, j] = (+ # utils.fma_packed_f32x2(+ # (acc_S_row_frg[k, j], acc_S_row_frg[k + 1, j]),+ # (self.scale_log2, self.scale_log2),+ # (minus_row_max_scaled, minus_row_max_scaled),+ # )+ # )+ # acc_S_row_frg[k, j] = utils.exp2f(acc_S_row_frg[k, j])+ # acc_S_row_frg[k + 1, j] = utils.exp2f(acc_S_row_frg[k + 1, j])+ acc_S_row_frg[k, j] = cute.arch.exp2(acc_S_row_frg[k, j])+ acc_S_row_frg[k + 1, j] = cute.arch.exp2(acc_S_row_frg[k + 1, j])+ acc_S_row_converted_frg[None, j].store(+ acc_S_row_frg[None, j].load().to(acc_S_row_converted.element_type)+ )+++def floor_if_packed(+ q_idx,+ qhead_per_kvhead: cutlass.Constexpr[int],+) -> cute.Tensor:+ """Convert q_idx to packed format for Pack-GQA."""+ if cutlass.const_expr(qhead_per_kvhead == 1):+ return q_idx+ return q_idx // qhead_per_kvhead+++def apply_score_mod_inner(+ score_tensor,+ index_tensor,+ score_mod: cutlass.Constexpr,+ batch_idx,+ head_idx,+ softmax_scale,+ vec_size: cutlass.Constexpr,+ qk_acc_dtype: cutlass.Constexpr,+ aux_tensors,+ fastdiv_mods,+ constant_q_idx: cutlass.Constexpr,+ qhead_per_kvhead: cutlass.Constexpr[int] = 1,+):+ """Shared implementation for applying score modification.++ Args:+ score_tensor: The scores to modify (acc_S for flash_fwd, tSrS_t2r for sm100)+ index_tensor: Index positions (tScS for flash_fwd, tScS_t2r for sm100)+ score_mod: The score modification function to apply+ batch_idx: Batch index+ head_idx: Head index+ softmax_scale: Scale to apply+ vec_size: Vector size for processing elements+ qk_acc_dtype: Data type for accumulator+ aux_tensors: Optional aux_tensors for FlexAttention+ fastdiv_mods: Tuple of (seqlen_q_divmod, seqlen_k_divmod) for wrapping+ constant_q_idx: If provided, use this constant for all q_idx values+ If None, compute q_idx per-element+ qhead_per_kvhead_packgqa: Pack-GQA replication factor. Divide q_idx by this+ when greater than 1 so score mods see logical heads.+ """+ n_vals = cutlass.const_expr(cute.size(score_tensor.shape))+ score_vec = cute.make_rmem_tensor(vec_size, qk_acc_dtype)+ kv_idx_vec = cute.make_rmem_tensor(vec_size, cutlass.Int32)++ # SSA values for batch (constant across all elements)+ batch_idx_ssa = utils.scalar_to_ssa(batch_idx, cutlass.Int32).broadcast_to((vec_size,))++ # Handle q_idx based on whether it's constant+ q_idx_vec = cute.make_rmem_tensor(vec_size, cutlass.Int32)++ # For Pack-GQA with non-constant q_idx, we need per-element head indices+ # since a thread my process multiple query head indices+ if cutlass.const_expr(qhead_per_kvhead > 1 and constant_q_idx is None):+ head_idx_vec = cute.make_rmem_tensor(vec_size, cutlass.Int32)++ for i in cutlass.range(0, n_vals, vec_size, unroll_full=True):+ for j in cutlass.range(vec_size, unroll_full=True):+ score_vec[j] = score_tensor[i + j] * softmax_scale++ # Extract head offset from packed q_idx for Pack-GQA+ if cutlass.const_expr(qhead_per_kvhead > 1 and constant_q_idx is None):+ q_idx_packed = index_tensor[i + j][0]+ # Building up the logical q_head idx: final_q_head = kv_head * qhead_per_kvhead + (q_physical % qhead_per_kvhead)+ q_idx_logical = q_idx_packed // qhead_per_kvhead+ head_offset = q_idx_packed - q_idx_logical * qhead_per_kvhead+ head_idx_vec[j] = head_idx * qhead_per_kvhead + head_offset++ # If we will do loads we mod, in order to not read OOB+ if cutlass.const_expr(aux_tensors is not None and fastdiv_mods is not None):+ if cutlass.const_expr(constant_q_idx is None):+ seqlen_q_divmod, seqlen_k_divmod = fastdiv_mods+ q_idx_floored = floor_if_packed(index_tensor[i + j][0], qhead_per_kvhead)+ _, q_idx_wrapped = seqlen_q_divmod.divmod(q_idx_floored)+ q_idx_vec[j] = q_idx_wrapped+ else:+ _, seqlen_k_divmod = fastdiv_mods++ _, kv_idx_wrapped = seqlen_k_divmod.divmod(index_tensor[i + j][1])+ kv_idx_vec[j] = kv_idx_wrapped+ else:+ # No bounds checking - direct indexing+ if constant_q_idx is None:+ q_idx_vec[j] = floor_if_packed(index_tensor[i + j][0], qhead_per_kvhead)+ kv_idx_vec[j] = index_tensor[i + j][1]++ # Convert to SSA for score_mod call+ score_ssa = score_vec.load()+ kv_idx_ssa = kv_idx_vec.load()+ if cutlass.const_expr(constant_q_idx is None):+ q_idx_ssa = q_idx_vec.load()+ else:+ # NB we do not apply Pack-GQA division here, as constant_q_idx is assumed to already be logical+ q_idx_const = constant_q_idx+ q_idx_ssa = utils.scalar_to_ssa(q_idx_const, cutlass.Int32).broadcast_to((vec_size,))++ # Compute head_idx_ssa: per-element for Pack-GQA with non-constant q_idx, constant otherwise+ if cutlass.const_expr(qhead_per_kvhead > 1 and constant_q_idx is None):+ head_idx_ssa = head_idx_vec.load()+ else:+ head_idx_ssa = utils.scalar_to_ssa(head_idx, cutlass.Int32).broadcast_to((vec_size,))++ aux_args = []+ if cutlass.const_expr(aux_tensors is not None):+ aux_args = aux_tensors++ post_mod_scores = score_mod(+ score_ssa,+ batch_idx_ssa,+ head_idx_ssa,+ q_idx=q_idx_ssa,+ kv_idx=kv_idx_ssa,+ aux_tensors=aux_args,+ )++ # Write back modified scores+ score_vec.store(post_mod_scores)+ for j in cutlass.range(vec_size, unroll_full=True):+ score_tensor[i + j] = score_vec[j]diff --git a/src/nvfp4_decode_kernel/_fa4/tile_scheduler.py b/src/nvfp4_decode_kernel/_fa4/tile_scheduler.pynew file mode 100644index 0000000..7c90b6f--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/tile_scheduler.py@@ -0,0 +1,695 @@+# Copyright (c) 2025, Tri Dao.++from typing import Optional, Tuple+from dataclasses import dataclass, fields++try:+ from typing import override+except ImportError: # Python < 3.12+ from typing_extensions import override++import cutlass+from cutlass._mlir import ir+import cutlass.cute as cute+from cutlass import Int32, const_expr++from . import utils as utils+from .fast_math import FastDivmod, clz+++class WorkTileInfo(cutlass.utils.WorkTileInfo):+ """Altered WorkTileInfo which includes four axes: (block, head, batch, split)"""++ @override+ def __new_from_mlir_values__(self, values: list[ir.Value]) -> "WorkTileInfo":+ assert len(values) == 5+ new_tile_idx = cutlass.new_from_mlir_values(self._tile_idx, values[:-1])+ new_is_valid_tile = cutlass.new_from_mlir_values(self._is_valid_tile, [values[-1]])+ return WorkTileInfo(new_tile_idx, new_is_valid_tile)+++@dataclass+class ParamsBase:+ def __extract_mlir_values__(self):+ all_fields = [getattr(self, field.name) for field in fields(self)]+ non_constexpr_fields = [f for f in all_fields if not isinstance(f, cutlass.Constexpr)]+ values, self._values_pos = [], []+ for obj in non_constexpr_fields:+ obj_values = cutlass.extract_mlir_values(obj)+ values += obj_values+ self._values_pos.append(len(obj_values))+ return values++ def __new_from_mlir_values__(self, values):+ all_fields = {field.name: getattr(self, field.name) for field in fields(self)}+ constexpr_fields = {n: f for n, f in all_fields.items() if isinstance(f, cutlass.Constexpr)}+ non_constexpr_fields = {+ n: f for n, f in all_fields.items() if not isinstance(f, cutlass.Constexpr)+ }+ for (name, field), n_items in zip(non_constexpr_fields.items(), self._values_pos):+ non_constexpr_fields[name] = cutlass.new_from_mlir_values(field, values[:n_items])+ values = values[n_items:]+ return self.__class__(**non_constexpr_fields, **constexpr_fields)+++@dataclass+class TileSchedulerArguments(ParamsBase):+ num_block: Int32+ num_head: Int32+ num_batch: Int32+ num_splits: Int32+ seqlen_k: Int32+ headdim: Int32+ headdim_v: Int32+ total_q: Int32+ tile_shape_mn: cutlass.Constexpr[Tuple[int, int]]+ cluster_shape_mn: cutlass.Constexpr[Tuple[int, int]] = (1, 1)+ mCuSeqlensQ: Optional[cute.Tensor] = None+ mSeqUsedQ: Optional[cute.Tensor] = None+ qhead_per_kvhead_packgqa: cutlass.Constexpr[int] = 1+ element_size: cutlass.Constexpr[int] = 2+ is_persistent: cutlass.Constexpr[bool] = False+ lpt: cutlass.Constexpr[bool] = False+ is_split_kv: cutlass.Constexpr[bool] = False+++class SingleTileScheduler:+ @dataclass+ class Params(ParamsBase):+ num_block: Int32+ num_head: Int32+ num_batch: Int32+ num_splits: Int32+ num_splits_divmod: FastDivmod+ is_split_kv: cutlass.Constexpr[bool] = False+ cluster_shape_mn: cutlass.Constexpr[Tuple[int, int]] = (1, 1)++ @staticmethod+ def create(+ args: TileSchedulerArguments, *, loc=None, ip=None+ ) -> "SingleTileScheduler.Params":+ return SingleTileScheduler.Params(+ args.num_block,+ args.num_head,+ args.num_batch,+ args.num_splits,+ FastDivmod.create(args.num_splits),+ args.is_split_kv,+ args.cluster_shape_mn,+ )++ def __init__(self, params: Params, blk_coord: cute.Coord, *, loc=None, ip=None):+ self.params = params+ self._blk_coord = blk_coord+ self._is_first_block = True+ self._loc = loc+ self._ip = ip++ @staticmethod+ def to_underlying_arguments(args: TileSchedulerArguments, *, loc=None, ip=None) -> Params:+ return SingleTileScheduler.Params.create(args, loc=loc, ip=ip)++ @staticmethod+ def create(params: Params, *, loc=None, ip=None) -> "SingleTileScheduler":+ blk_coord = cute.arch.block_idx()+ return SingleTileScheduler(params, blk_coord, loc=loc, ip=ip)++ # called by host+ @staticmethod+ def get_grid_shape(+ params: Params,+ *,+ loc=None,+ ip=None,+ ) -> Tuple[Int32, Int32, Int32]:+ # TODO: this hard-codes the fact that we only use cluster = (1, 1) or (2, 1)+ assert params.cluster_shape_mn[1] == 1, "Only cluster_shape_mn[1] == 1 is supported"+ return (+ cute.round_up(params.num_block, params.cluster_shape_mn[0]),+ params.num_head * params.num_splits,+ params.num_batch,+ )++ def get_current_work(self, *, loc=None, ip=None) -> WorkTileInfo:+ block_idx, head_idx, batch_idx = self._blk_coord+ if const_expr(self.params.is_split_kv):+ head_idx, split_idx = self.params.num_splits_divmod.divmod(head_idx)+ else:+ split_idx = Int32(0)+ return WorkTileInfo(+ (block_idx, head_idx, batch_idx, split_idx),+ self._is_first_block,+ )++ def initial_work_tile_info(self, *, loc=None, ip=None):+ return self.get_current_work(loc=loc, ip=ip)++ def prefetch_next_work(self, *, loc=None, ip=None):+ pass++ def advance_to_next_work(self, *, loc=None, ip=None):+ self._is_first_block = False++ def __extract_mlir_values__(self):+ values, self._values_pos = [], []+ for obj in [self.params, self._blk_coord]:+ obj_values = cutlass.extract_mlir_values(obj)+ values += obj_values+ self._values_pos.append(len(obj_values))+ return values++ def __new_from_mlir_values__(self, values):+ obj_list = []+ for obj, n_items in zip([self.params, self._blk_coord], self._values_pos):+ obj_list.append(cutlass.new_from_mlir_values(obj, values[:n_items]))+ values = values[n_items:]+ return SingleTileScheduler(*(tuple(obj_list)), loc=self._loc)+++class StaticPersistentTileScheduler:+ @dataclass+ class Params(ParamsBase):+ num_block_divmod: FastDivmod+ num_head_divmod: FastDivmod+ total_blocks: Int32++ @staticmethod+ def create(+ args: TileSchedulerArguments, *, loc=None, ip=None+ ) -> "StaticPersistentTileScheduler.Params":+ total_blocks = args.num_block * args.num_head * args.num_batch+ return StaticPersistentTileScheduler.Params(+ FastDivmod.create(args.num_block), FastDivmod.create(args.num_head), total_blocks+ )++ def __init__(self, params: Params, tile_idx: Int32, *, loc=None, ip=None):+ self.params = params+ self._tile_idx = tile_idx+ self._loc = loc+ self._ip = ip++ @staticmethod+ def to_underlying_arguments(args: TileSchedulerArguments, *, loc=None, ip=None) -> Params:+ return StaticPersistentTileScheduler.Params.create(args, loc=loc, ip=ip)++ @staticmethod+ def create(params: Params, *, loc=None, ip=None) -> "StaticPersistentTileScheduler":+ tile_idx = cute.arch.block_idx()[0]+ return StaticPersistentTileScheduler(params, tile_idx, loc=loc, ip=ip)++ # called by host+ @staticmethod+ def get_grid_shape(+ params: Params,+ *,+ loc=None,+ ip=None,+ ) -> Tuple[Int32, Int32, Int32]:+ hardware_info = cutlass.utils.HardwareInfo()+ sm_count = hardware_info.get_device_multiprocessor_count()+ return (cutlass.min(sm_count, params.total_blocks), Int32(1), Int32(1))++ # @cute.jit+ def get_current_work(self, *, loc=None, ip=None) -> WorkTileInfo:+ hn_idx, block_idx = self.params.num_block_divmod.divmod(self._tile_idx)+ batch_idx, head_idx = self.params.num_head_divmod.divmod(hn_idx)+ is_valid = self._tile_idx < self.params.total_blocks+ # if cute.arch.thread_idx()[0] == 0:+ # cute.printf("TileScheduler: tile_idx=%d, hn_idx=%d, block_idx=%d, batch_idx=%d, head_idx=%d, is_valid=%d", self._tile_idx, hn_idx, block_idx, batch_idx, head_idx, is_valid)+ return WorkTileInfo(+ (Int32(block_idx), Int32(head_idx), Int32(batch_idx), Int32(0)), is_valid+ )++ def initial_work_tile_info(self, *, loc=None, ip=None):+ return self.get_current_work(loc=loc, ip=ip)++ def prefetch_next_work(self, *, loc=None, ip=None):+ pass++ def advance_to_next_work(self, *, loc=None, ip=None):+ self._tile_idx += cute.arch.grid_dim()[0]++ def __extract_mlir_values__(self):+ values, self._values_pos = [], []+ for obj in [self.params, self._tile_idx]:+ obj_values = cutlass.extract_mlir_values(obj)+ values += obj_values+ self._values_pos.append(len(obj_values))+ return values++ def __new_from_mlir_values__(self, values):+ obj_list = []+ for obj, n_items in zip(+ [self.params, self._tile_idx],+ self._values_pos,+ ):+ obj_list.append(cutlass.new_from_mlir_values(obj, values[:n_items]))+ values = values[n_items:]+ return StaticPersistentTileScheduler(*(tuple(obj_list)), loc=self._loc)+++class SingleTileLPTScheduler:+ @dataclass+ class Params(ParamsBase):+ total_blocks: Int32+ num_splits: Int32+ num_block_divmod: FastDivmod+ num_head_divmod: FastDivmod+ l2_minor_divmod: FastDivmod+ l2_major_divmod: FastDivmod+ l2_minor_residual_divmod: FastDivmod+ num_hb_quotient: Int32+ is_split_kv: cutlass.Constexpr[bool] = False++ @staticmethod+ @cute.jit+ def create(+ args: TileSchedulerArguments, *, loc=None, ip=None+ ) -> "SingleTileLPTScheduler.Params":+ # cute.printf(args.num_block, args.num_head, args.num_batch, args.seqlen_k, args.headdim, args.headdim_v, args.total_q, args.tile_shape_mn, args.qhead_per_kvhead_packgqa, args.element_size)+ size_one_kv_head = args.seqlen_k * (args.headdim + args.headdim_v) * args.element_size+ size_one_head = size_one_kv_head+ size_l2 = 50 * 1024 * 1024 # 40 MB for K & V+ # Swizzle is the size of each "section". Round swizzle to a power of 2+ # Need to be careful about the case where only one head will fit+ # swizzle is how many heads can fit in L2+ # swizzle = 1 if size_l2 < size_one_head else (size_l2 // size_one_head)+ # Seems faster if swizzle if a power of 2+ log2_floor = lambda n: 31 - clz(n)+ swizzle = 1 if size_l2 < size_one_head else (1 << log2_floor(size_l2 // size_one_head))+ # swizzle = 1 if size_l2 < size_one_head else (size_l2 // size_one_head)+ # If we're in the last section (called residual), we don't want to divide by+ # swizzle. Instead we want to divide by the remainder.+ num_hb_quotient = (args.num_head * args.num_batch) // swizzle+ num_hb_remainder = (args.num_head * args.num_batch) % swizzle+ return SingleTileLPTScheduler.Params(+ total_blocks=args.num_block * args.num_head * args.num_batch,+ num_block_divmod=FastDivmod.create(args.num_block),+ num_head_divmod=FastDivmod.create(args.num_head),+ l2_minor_divmod=FastDivmod.create(swizzle),+ l2_major_divmod=FastDivmod.create(swizzle * args.num_block),+ l2_minor_residual_divmod=FastDivmod.create(+ max(num_hb_remainder, 1)+ ), # don't divide by 0+ num_hb_quotient=Int32(num_hb_quotient),+ num_splits=args.num_splits,+ is_split_kv=args.is_split_kv,+ )++ def __init__(self, params: Params, tile_idx: Int32, split_idx: Int32, *, loc=None, ip=None):+ self.params = params+ self._tile_idx = tile_idx+ self._split_idx = split_idx+ self._loc = loc+ self._ip = ip++ @staticmethod+ def to_underlying_arguments(args: TileSchedulerArguments, *, loc=None, ip=None) -> Params:+ return SingleTileLPTScheduler.Params.create(args, loc=loc, ip=ip)++ @staticmethod+ @cute.jit+ def create(params: Params, *, loc=None, ip=None) -> "SingleTileLPTScheduler":+ tile_idx, split_idx, _ = cute.arch.block_idx()+ return SingleTileLPTScheduler(params, tile_idx, split_idx, loc=loc, ip=ip)++ # called by host+ @staticmethod+ def get_grid_shape(+ params: Params,+ *,+ loc=None,+ ip=None,+ ) -> Tuple[Int32, Int32, Int32]:+ return (params.total_blocks, params.num_splits, Int32(1))++ @cute.jit+ def get_current_work(self, *, loc=None, ip=None) -> WorkTileInfo:+ params = self.params+ # Implement LPT scheduling coordinate calculation+ bidhb, l2_mod = params.l2_major_divmod.divmod(self._tile_idx)+ # If we're in the last section (called residual), we don't want to divide by+ # swizzle. Instead we want to divide by the remainder.+ block, bidhb_residual = 0, 0+ if bidhb < params.num_hb_quotient:+ block, bidhb_residual = params.l2_minor_divmod.divmod(l2_mod)+ else:+ block, bidhb_residual = params.l2_minor_residual_divmod.divmod(l2_mod)+ bidhb_actual = bidhb * params.l2_minor_divmod.divisor + bidhb_residual+ batch_idx, head_idx = params.num_head_divmod.divmod(bidhb_actual)+ # Longest-processing-time-first+ block = params.num_block_divmod.divisor - 1 - block+ is_valid = self._tile_idx < params.total_blocks+ return WorkTileInfo(+ (Int32(block), Int32(head_idx), Int32(batch_idx), Int32(self._split_idx)), is_valid+ )++ def initial_work_tile_info(self, *, loc=None, ip=None):+ return self.get_current_work(loc=loc, ip=ip)++ def prefetch_next_work(self, *, loc=None, ip=None):+ pass++ def advance_to_next_work(self, *, loc=None, ip=None):+ # Single tile scheduler - set to invalid tile_idx to indicate no more work+ self._tile_idx = self.params.total_blocks++ def __extract_mlir_values__(self):+ values, self._values_pos = [], []+ for obj in [self.params, self._tile_idx, self._split_idx]:+ obj_values = cutlass.extract_mlir_values(obj)+ values += obj_values+ self._values_pos.append(len(obj_values))+ return values++ def __new_from_mlir_values__(self, values):+ obj_list = []+ for obj, n_items in zip([self.params, self._tile_idx, self._split_idx], self._values_pos):+ obj_list.append(cutlass.new_from_mlir_values(obj, values[:n_items]))+ values = values[n_items:]+ return self.__class__(*(tuple(obj_list)), loc=self._loc)+++class SingleTileLPTBwdScheduler:+ @dataclass+ class Params(ParamsBase):+ total_blocks: Int32+ num_head_divmod: FastDivmod+ l2_minor_divmod: FastDivmod+ l2_major_divmod: FastDivmod+ l2_minor_residual_divmod: FastDivmod+ num_hb_quotient: Int32+ cluster_shape_mn: cutlass.Constexpr[Tuple[int, int]] = (1, 1)++ @staticmethod+ @cute.jit+ def create(+ args: TileSchedulerArguments, *, loc=None, ip=None+ ) -> "SingleTileLPTBwdScheduler.Params":+ swizzle = 8+ # If we're in the last section (called residual), we don't want to divide by+ # swizzle. Instead we want to divide by the remainder.+ num_hb_quotient = (args.num_head * args.num_batch) // swizzle+ num_hb_remainder = (args.num_head * args.num_batch) % swizzle+ num_block = cute.ceil_div(args.num_block, args.cluster_shape_mn[0])+ return SingleTileLPTBwdScheduler.Params(+ total_blocks=(num_block * args.cluster_shape_mn[0])+ * args.num_head+ * args.num_batch,+ num_head_divmod=FastDivmod.create(args.num_head),+ l2_minor_divmod=FastDivmod.create(swizzle),+ l2_major_divmod=FastDivmod.create(swizzle * num_block),+ l2_minor_residual_divmod=FastDivmod.create(+ max(num_hb_remainder, 1)+ ), # don't divide by 0+ num_hb_quotient=Int32(num_hb_quotient),+ cluster_shape_mn=args.cluster_shape_mn,+ )++ def __init__(self, params: Params, tile_idx: Int32, *, loc=None, ip=None):+ self.params = params+ self._tile_idx = tile_idx+ self._loc = loc+ self._ip = ip++ @staticmethod+ def to_underlying_arguments(args: TileSchedulerArguments, *, loc=None, ip=None) -> Params:+ return SingleTileLPTBwdScheduler.Params.create(args, loc=loc, ip=ip)++ @staticmethod+ @cute.jit+ def create(params: Params, *, loc=None, ip=None) -> "SingleTileLPTBwdScheduler":+ tile_idx = cute.arch.block_idx()[0]+ return SingleTileLPTBwdScheduler(params, tile_idx, loc=loc, ip=ip)++ # called by host+ @staticmethod+ def get_grid_shape(+ params: Params,+ *,+ loc=None,+ ip=None,+ ) -> Tuple[Int32, Int32, Int32]:+ return (params.total_blocks, Int32(1), Int32(1))++ @cute.jit+ def get_current_work(self, *, loc=None, ip=None) -> cutlass.utils.WorkTileInfo:+ cluster_idx = self._tile_idx // self.params.cluster_shape_mn[0]+ params = self.params+ # Implement LPT scheduling coordinate calculation+ bidhb, l2_mod = params.l2_major_divmod.divmod(cluster_idx)+ # If we're in the last section (called residual), we don't want to divide by+ # swizzle. Instead we want to divide by the remainder.+ block, bidhb_residual = 0, 0+ if bidhb < params.num_hb_quotient:+ block, bidhb_residual = params.l2_minor_divmod.divmod(l2_mod)+ else:+ block, bidhb_residual = params.l2_minor_residual_divmod.divmod(l2_mod)+ bidhb_actual = bidhb * params.l2_minor_divmod.divisor + bidhb_residual+ batch_idx, head_idx = params.num_head_divmod.divmod(bidhb_actual)+ is_valid = self._tile_idx < params.total_blocks+ bidx_in_cluster = cute.arch.block_in_cluster_idx()+ block = block * params.cluster_shape_mn[0] + bidx_in_cluster[0]+ return WorkTileInfo((Int32(block), Int32(head_idx), Int32(batch_idx), Int32(0)), is_valid)++ def initial_work_tile_info(self, *, loc=None, ip=None):+ return self.get_current_work(loc=loc, ip=ip)++ def prefetch_next_work(self, *, loc=None, ip=None):+ pass++ def advance_to_next_work(self, *, loc=None, ip=None):+ # Single tile scheduler - set to invalid tile_idx to indicate no more work+ self._tile_idx = self.params.total_blocks++ def __extract_mlir_values__(self):+ values, self._values_pos = [], []+ for obj in [self.params, self._tile_idx]:+ obj_values = cutlass.extract_mlir_values(obj)+ values += obj_values+ self._values_pos.append(len(obj_values))+ return values++ def __new_from_mlir_values__(self, values):+ obj_list = []+ for obj, n_items in zip([self.params, self._tile_idx], self._values_pos):+ obj_list.append(cutlass.new_from_mlir_values(obj, values[:n_items]))+ values = values[n_items:]+ return self.__class__(*(tuple(obj_list)), loc=self._loc)+++class SingleTileVarlenScheduler:+ @dataclass+ class Params(ParamsBase):+ num_head: Int32+ num_batch: Int32+ total_q: Int32+ num_splits: Int32+ max_kvblock_in_l2: Int32+ tile_shape_mn: cutlass.Constexpr[Tuple[int, int]]+ mCuSeqlensQ: Optional[cute.Tensor] = None+ mSeqUsedQ: Optional[cute.Tensor] = None+ qhead_per_kvhead_packgqa: cutlass.Constexpr[int] = 1+ lpt: cutlass.Constexpr[bool] = False+ is_split_kv: cutlass.Constexpr[bool] = False++ @staticmethod+ @cute.jit+ def create(+ args: TileSchedulerArguments, *, loc=None, ip=None+ ) -> "SingleTileVarlenScheduler.Params":+ size_l2 = 50 * 1024 * 1024 # 50 MB for K & V+ max_kvblock_in_l2 = size_l2 // (+ (args.headdim + args.headdim_v) * args.element_size * args.tile_shape_mn[1]+ )+ assert args.mCuSeqlensQ is not None or args.mSeqUsedQ is not None, (+ "At least one of mCuSeqlensQ or mSeqUsedQ must be provided"+ )+ return SingleTileVarlenScheduler.Params(+ num_head=args.num_head,+ num_batch=args.num_batch,+ total_q=args.total_q,+ num_splits=args.num_splits,+ max_kvblock_in_l2=max_kvblock_in_l2,+ tile_shape_mn=args.tile_shape_mn,+ mCuSeqlensQ=args.mCuSeqlensQ,+ mSeqUsedQ=args.mSeqUsedQ,+ qhead_per_kvhead_packgqa=args.qhead_per_kvhead_packgqa,+ lpt=args.lpt,+ is_split_kv=args.is_split_kv,+ )++ def __init__(self, params: Params, tile_idx: Int32, split_idx: Int32, *, loc=None, ip=None):+ self.params = params+ self._tile_idx = tile_idx+ self._split_idx = split_idx+ self._is_first_block = True+ self._loc = loc+ self._ip = ip++ @staticmethod+ def to_underlying_arguments(args: TileSchedulerArguments, *, loc=None, ip=None) -> Params:+ return SingleTileVarlenScheduler.Params.create(args, loc=loc, ip=ip)++ @staticmethod+ def create(params: Params, *, loc=None, ip=None) -> "SingleTileVarlenScheduler":+ tile_idx, split_idx, _ = cute.arch.block_idx()+ return SingleTileVarlenScheduler(params, tile_idx, split_idx, loc=loc, ip=ip)++ # called by host+ @staticmethod+ def get_grid_shape(+ params: Params,+ *,+ loc=None,+ ip=None,+ ) -> Tuple[Int32, Int32, Int32]:+ total_blocks_max = (+ params.total_q + params.num_batch * (params.tile_shape_mn[0] - 1)+ ) // params.tile_shape_mn[0]+ return (total_blocks_max * params.num_head, params.num_splits, Int32(1))++ @cute.jit+ def _get_num_m_blocks(self, lane: Int32, bidb_start: Int32) -> Int32:+ params = self.params+ batch_idx = lane + bidb_start+ if cutlass.const_expr(params.mSeqUsedQ is not None):+ seqlen = Int32(0)+ if batch_idx < params.num_batch:+ seqlen = params.mSeqUsedQ[batch_idx]+ else:+ assert params.mCuSeqlensQ is not None+ cur_cu_seqlen = Int32(0)+ if batch_idx <= params.num_batch:+ cur_cu_seqlen = params.mCuSeqlensQ[batch_idx]+ next_cu_seqlen = cute.arch.shuffle_sync_down(cur_cu_seqlen, offset=1)+ seqlen = next_cu_seqlen - cur_cu_seqlen+ if cutlass.const_expr(params.qhead_per_kvhead_packgqa > 1):+ seqlen *= params.qhead_per_kvhead_packgqa+ return (+ cute.ceil_div(seqlen, params.tile_shape_mn[0])+ if batch_idx < params.num_batch and lane < cute.arch.WARP_SIZE - 1+ else Int32(0)+ )++ @cute.jit+ def get_current_work(self, *, loc=None, ip=None) -> WorkTileInfo:+ params = self.params+ lane_idx = cute.arch.lane_idx()+ num_m_blocks = self._get_num_m_blocks(lane_idx, bidb_start=0)+ num_m_blocks_cumulative = utils.warp_prefix_sum(num_m_blocks, lane_idx)+ # Total number of blocks for the next 31 batches+ m_blocks_in_group = cute.arch.shuffle_sync(num_m_blocks_cumulative, cute.arch.WARP_SIZE - 1)+ # Same for all lanes+ group_end_tile = m_blocks_in_group * params.num_head+ # if cute.arch.thread_idx()[0] == 128 + 31: cute.printf("SingleTileVarlenScheduler: tile_idx=%d, group_end_tile = %d, num_m_blocks=%d, num_m_blocks_cumulative = %d, m_blocks_in_group = %d", self._tile_idx, group_end_tile, num_m_blocks, num_m_blocks_cumulative, m_blocks_in_group)+ block, head_idx, batch_idx = Int32(0), Int32(0), Int32(0)+ next_tile_idx = self._tile_idx+ while group_end_tile <= next_tile_idx:+ batch_idx += cute.arch.WARP_SIZE - 1+ if batch_idx >= params.num_batch:+ batch_idx = Int32(params.num_batch)+ group_end_tile = next_tile_idx + 1+ else:+ num_m_blocks = self._get_num_m_blocks(lane_idx, bidb_start=batch_idx)+ num_m_blocks_cumulative = utils.warp_prefix_sum(num_m_blocks, lane_idx)+ m_blocks_in_group = cute.arch.shuffle_sync(+ num_m_blocks_cumulative, cute.arch.WARP_SIZE - 1+ )+ group_end_tile += m_blocks_in_group * params.num_head+ is_valid = False+ if batch_idx >= params.num_batch:+ block, head_idx, batch_idx = Int32(0), Int32(0), Int32(params.num_batch)+ else:+ group_start_tile = group_end_tile - m_blocks_in_group * params.num_head+ # if cute.arch.thread_idx()[0] == 128 + 31: cute.printf("SingleTileVarlenScheduler: tile_idx=%d, group_end_tile = %d, num_m_blocks=%d, batch_idx = %d", self._tile_idx, group_end_tile, num_m_blocks, batch_idx)+ # The next problem to process is the first one that does not have ending tile position+ # that is greater than or equal to tile index.+ batch_idx_in_group = cute.arch.popc(+ cute.arch.vote_ballot_sync(+ group_start_tile + num_m_blocks_cumulative * params.num_head <= next_tile_idx+ )+ )+ batch_idx += batch_idx_in_group+ num_m_blocks_prev_lane = (+ 0+ if batch_idx_in_group == 0+ else cute.arch.shuffle_sync(num_m_blocks_cumulative, batch_idx_in_group - 1)+ )+ num_m_blocks = cute.arch.shuffle_sync(num_m_blocks, batch_idx_in_group)+ mh_block = next_tile_idx - group_start_tile - num_m_blocks_prev_lane * params.num_head+ if cutlass.const_expr(params.lpt):+ # This is a version of the SingleTileLPTScheduler, complicated by the fact that+ # the seqlen can vary per batch.+ # TODO: is there any case where num_m_blocks is 0?+ # TODO: by right we should read the seqlen_kv but we're assuming seqlen_q == seqlen_k here+ num_n_blocks = (+ num_m_blocks+ * params.tile_shape_mn[0]+ // params.qhead_per_kvhead_packgqa+ // params.tile_shape_mn[1]+ )+ # nheads_in_l2 = min(max(self.max_kvblock_in_l2 // num_n_blocks, 1), self.num_head)+ # Seems faster to have this be a power of 2+ nheads_in_l2 = (+ 16+ if num_n_blocks * 16 <= params.max_kvblock_in_l2+ else (+ 8+ if num_n_blocks * 8 <= params.max_kvblock_in_l2+ else (+ 4+ if num_n_blocks * 4 <= params.max_kvblock_in_l2+ else (2 if num_n_blocks * 2 <= params.max_kvblock_in_l2 else 1)+ )+ )+ )+ nheads_in_l2 = min(nheads_in_l2, params.num_head)+ mh_in_l2 = nheads_in_l2 * num_m_blocks+ section_idx = mh_block // mh_in_l2+ l2_mod = mh_block - section_idx * mh_in_l2+ # Deal with tail section+ nheads_in_this_section = (+ nheads_in_l2+ if nheads_in_l2 * (section_idx + 1) <= params.num_head+ else params.num_head - section_idx * nheads_in_l2+ )+ block = l2_mod // nheads_in_this_section+ head_idx_residual = l2_mod - block * nheads_in_this_section+ head_idx = section_idx * nheads_in_l2 + head_idx_residual+ block = num_m_blocks - 1 - block+ else:+ head_idx = mh_block // num_m_blocks+ block = mh_block - head_idx * num_m_blocks+ is_valid = self._is_first_block and batch_idx < params.num_batch+ # if cute.arch.thread_idx()[0] == 128: cute.printf("SingleTileVarlenScheduler: tile_idx=%d, batch_idx=%d, head_idx=%d, block=%d, is_valid = %d", self._tile_idx, batch_idx, head_idx, block, is_valid)+ split_idx = self._split_idx if const_expr(params.is_split_kv) else Int32(0)+ return WorkTileInfo((Int32(block), Int32(head_idx), Int32(batch_idx), split_idx), is_valid)++ def initial_work_tile_info(self, *, loc=None, ip=None):+ return self.get_current_work(loc=loc, ip=ip)++ def prefetch_next_work(self, *, loc=None, ip=None):+ pass++ def advance_to_next_work(self, *, loc=None, ip=None):+ # Single tile scheduler - set to invalid tile_idx to indicate no more work+ self._is_first_block = False++ def __extract_mlir_values__(self):+ values, self._values_pos = [], []+ for obj in [self.params, self._tile_idx, self._split_idx]:+ obj_values = cutlass.extract_mlir_values(obj)+ values += obj_values+ self._values_pos.append(len(obj_values))+ return values++ def __new_from_mlir_values__(self, values):+ obj_list = []+ for obj, n_items in zip(+ [self.params, self._tile_idx, self._split_idx],+ self._values_pos,+ ):+ obj_list.append(cutlass.new_from_mlir_values(obj, values[:n_items]))+ values = values[n_items:]+ return SingleTileVarlenScheduler(*(tuple(obj_list)), loc=self._loc)diff --git a/src/nvfp4_decode_kernel/_fa4/utils.py b/src/nvfp4_decode_kernel/_fa4/utils.pynew file mode 100644index 0000000..ca25537--- /dev/null+++ b/src/nvfp4_decode_kernel/_fa4/utils.py@@ -0,0 +1,809 @@+# Copyright (c) 2025, Tri Dao.++import math+import hashlib+import inspect+import re+from typing import Type, Callable, Optional, Tuple, overload+from functools import partial++import cutlass+import cutlass.cute as cute++from cutlass import Float32, Int32, const_expr+from cutlass.cutlass_dsl import T, dsl_user_op+from cutlass._mlir.dialects import nvvm, llvm+from cutlass.cute.runtime import from_dlpack+++fma_packed_f32x2 = partial(cute.arch.fma_packed_f32x2, rnd="rn")+mul_packed_f32x2 = partial(cute.arch.mul_packed_f32x2, rnd="rn")+add_packed_f32x2 = partial(cute.arch.add_packed_f32x2, rnd="rn")+sub_packed_f32x2 = partial(+ cute.arch.calc_packed_f32x2_op,+ src_c=None,+ calc_func=nvvm.sub_packed_f32x2,+ rnd="rn",+)++def hash_callable(func: Callable) -> str:+ """Hash a callable based on the source code or bytecode and closure values."""+ if hasattr(func, "__wrapped__"):+ # cute.jit returns a wrapper whose repr/closure changes per compile; hash the undecorated function.+ base_func = func.__wrapped__+ func = base_func++ try:+ data = inspect.getsource(func).encode()+ except (OSError, TypeError):+ if hasattr(func, "__code__") and func.__code__ is not None:+ data = func.__code__.co_code+ else:+ data = repr(func).encode()++ hasher = hashlib.sha256(data)++ if hasattr(func, "__closure__") and func.__closure__ is not None:+ for idx, cell in enumerate(func.__closure__):+ cell_value = cell.cell_contents+ hasher.update(repr(cell_value).encode())++ return hasher.hexdigest()+++def create_softcap_scoremod(softcap_val):+ inv_softcap = 1.0 / softcap_val++ @cute.jit+ def scoremod_premask_fn(acc_S_SSA, batch_idx, head_idx, q_idx, kv_idx, buffers):+ scores = acc_S_SSA * inv_softcap+ return scores * cute.math.tanh(scores, fastmath=True)++ return scoremod_premask_fn++def convert_from_dlpack(x, leading_dim, alignment=16, divisibility=1) -> cute.Tensor:+ return (+ from_dlpack(x, assumed_align=alignment)+ .mark_layout_dynamic(leading_dim=leading_dim)+ .mark_compact_shape_dynamic(+ mode=leading_dim, stride_order=x.dim_order(), divisibility=divisibility+ )+ )+++def make_tiled_copy_A(+ copy_atom: cute.CopyAtom, tiled_mma: cute.TiledMma, swapAB: cutlass.Constexpr[bool] = False+) -> cute.TiledCopy:+ if const_expr(swapAB):+ return cute.make_tiled_copy_B(copy_atom, tiled_mma)+ else:+ return cute.make_tiled_copy_A(copy_atom, tiled_mma)+++def make_tiled_copy_B(+ copy_atom: cute.CopyAtom, tiled_mma: cute.TiledMma, swapAB: cutlass.Constexpr[bool] = False+) -> cute.TiledCopy:+ if const_expr(swapAB):+ return cute.make_tiled_copy_A(copy_atom, tiled_mma)+ else:+ return cute.make_tiled_copy_B(copy_atom, tiled_mma)+++def mma_make_fragment_A(+ smem: cute.Tensor, thr_mma: cute.ThrMma, swapAB: cutlass.Constexpr[bool] = False+) -> cute.Tensor:+ if const_expr(swapAB):+ return mma_make_fragment_B(smem, thr_mma)+ else:+ return thr_mma.make_fragment_A(thr_mma.partition_A(smem))+++def mma_make_fragment_B(+ smem: cute.Tensor, thr_mma: cute.ThrMma, swapAB: cutlass.Constexpr[bool] = False+) -> cute.Tensor:+ if const_expr(swapAB):+ return mma_make_fragment_A(smem, thr_mma)+ else:+ return thr_mma.make_fragment_B(thr_mma.partition_B(smem))+++def get_smem_store_atom(+ arch: cutlass.Constexpr[int], element_type: Type[cute.Numeric], transpose: bool = False+) -> cute.CopyAtom:+ if const_expr(arch < 90 or element_type.width != 16):+ return cute.make_copy_atom(+ cute.nvgpu.CopyUniversalOp(),+ element_type,+ num_bits_per_copy=2 * element_type.width,+ )+ else:+ return cute.make_copy_atom(+ cute.nvgpu.warp.StMatrix8x8x16bOp(transpose=transpose, num_matrices=4),+ element_type,+ )+++def warp_reduce(+ val: cute.TensorSSA | cute.Numeric,+ op: Callable,+ width: cutlass.Constexpr[int] = cute.arch.WARP_SIZE,+) -> cute.TensorSSA | cute.Numeric:+ if const_expr(isinstance(val, cute.TensorSSA)):+ res = cute.make_rmem_tensor(val.shape, val.dtype)+ res.store(val)+ for i in cutlass.range_constexpr(cute.size(val.shape)):+ res[i] = warp_reduce(res[i], op, width)+ return res.load()+ else:+ for i in cutlass.range_constexpr(int(math.log2(width))):+ val = op(val, cute.arch.shuffle_sync_bfly(val, offset=1 << i))+ return val+++def convert_layout_acc_mn(acc_layout: cute.Layout, transpose: bool = False) -> cute.Layout:+ """+ For Sm80, convert ((2, 2), MMA_M, MMA_N, ...) to ((2, MMA_M), (2, MMA_N), ...).+ For Sm90, convert ((2, 2, V), MMA_M, MMA_N, ...) to ((2, MMA_M), (2, V, MMA_N), ...).+ """+ acc_layout_col_major = cute.make_layout(acc_layout.shape)+ shape = (+ (acc_layout_col_major.shape[0][1], acc_layout_col_major.shape[1]), # MMA_M+ (+ acc_layout_col_major.shape[0][0],+ *acc_layout_col_major.shape[0][2:],+ acc_layout_col_major.shape[2],+ ), # MMA_N+ *acc_layout_col_major.shape[3:],+ )+ stride = (+ (acc_layout_col_major.stride[0][1], acc_layout_col_major.stride[1]), # MMA_M+ (+ acc_layout_col_major.stride[0][0],+ *acc_layout_col_major.stride[0][2:],+ acc_layout_col_major.stride[2],+ ), # MMA_N+ *acc_layout_col_major.stride[3:],+ )+ if const_expr(transpose):+ shape = (shape[1], shape[0], *shape[2:])+ stride = (stride[1], stride[0], *stride[2:])+ acc_layout_mn = cute.make_layout(shape, stride=stride)+ return cute.composition(acc_layout, acc_layout_mn)+++def make_acc_tensor_mn_view(acc: cute.Tensor, transpose: bool = False) -> cute.Tensor:+ return cute.make_tensor(acc.iterator, convert_layout_acc_mn(acc.layout, transpose=transpose))+++def convert_layout_acc_frgA(acc_layout: cute.Layout) -> cute.Layout:+ # For back to back gemm, convert layout of acc0 to gemm 1 accept layout.+ # For Sm80, as the mma instruction shape is 16x8x16, we need to convert from (4, MMA_M, MMA_N) to ((4, 2), MMA_M, MMA_N / 2)+ # For Sm90, FP16/BF16, convert acc_layout from ((2, 2, N / 8), MMA_M, MMA_N) to ((2, 2, 2), MMA_M, (N / 16, MMA_N))+ # TODO: Sm90 FP8+ if const_expr(cute.rank(acc_layout.shape[0]) == 3): # Sm90+ l = cute.logical_divide(+ acc_layout, ((None, None, 2), None, None)+ ) # ((2, 2, (2, N / 16)), MMA_M, MMA_N)+ rA_mma_view = cute.make_layout(+ (+ (l.shape[0][0], l.shape[0][1], l.shape[0][2][0]),+ l.shape[1],+ (l.shape[0][2][1], l.shape[2]),+ ),+ stride=(+ (l.stride[0][0], l.stride[0][1], l.stride[0][2][0]),+ l.stride[1],+ (l.stride[0][2][1], l.stride[2]),+ ),+ )+ else: # Sm80+ # (4, MMA_M, MMA_N) -> (4, MMA_M, (2, MMA_N / 2))+ l = cute.logical_divide(acc_layout, (None, None, 2))+ rA_mma_view = cute.make_layout(+ (+ (l.shape[0], l.shape[2][0]),+ l.shape[1],+ l.shape[2][1],+ ),+ stride=(+ (l.stride[0], l.stride[2][0]),+ l.stride[1],+ l.stride[2][1],+ ),+ )+ return rA_mma_view+++def make_acc_tensor_frgA_view(acc: cute.Tensor) -> cute.Tensor:+ return cute.make_tensor(acc.iterator, convert_layout_acc_frgA(acc.layout))+++def select(a: cute.Tensor, mode: list[int]) -> cute.Tensor:+ return cute.make_tensor(a.iterator, cute.select(a.layout, mode))+++def transpose_view(a: cute.Tensor) -> cute.Tensor:+ """Transpose the first two dimensions of a tensor on smem."""+ shape = (a.shape[1], a.shape[0], *a.shape[2:])+ order = (1, 0, *range(2, cute.rank(a)))+ return cute.composition(a, cute.make_ordered_layout(shape, order=order))+ # stride = (a.layout.stride[1], a.layout.stride[0], *a.layout.stride[2:])+ # return cute.make_tensor(a.iterator, cute.make_layout(shape, stride=stride))+++def parse_swizzle_from_pointer(ptr: cute.Pointer) -> cute.Swizzle:+ """Extract swizzle parameters from a pointer's swizzle_type.++ The swizzle_type string has the form '!cute.swizzle<"S<b,m,s>">' where+ b, m, s are the swizzle parameters (bits, base, shift).++ Returns:+ A cute.Swizzle object constructed from the extracted parameters++ Raises:+ ValueError: If the swizzle_type string cannot be parsed+ """+ # Ideally there should be a better API to get swizzle parameters, but we'll just parse+ # the string here.+ swizzle_str = str(ptr.type.swizzle_type)+ # Extract the inner part "S<b,m,s>"+ match = re.search(r'S<(\d+),(\d+),(\d+)>', swizzle_str)+ if match:+ b, m, s = int(match.group(1)), int(match.group(2)), int(match.group(3))+ return cute.make_swizzle(b, m, s)+ else:+ raise ValueError(f"Could not parse swizzle_type: {swizzle_str}")+++def exp2f(x: cute.TensorSSA | Float32) -> cute.TensorSSA | Float32:+ """exp2f calculation for both vector and scalar.+ :param x: input value+ :type x: cute.TensorSSA or Float32+ :return: exp2 value+ :rtype: cute.TensorSSA or Float32+ """+ if const_expr(isinstance(x, cute.TensorSSA)):+ res = cute.make_rmem_tensor(x.shape, Float32)+ res.store(x)+ for i in cutlass.range_constexpr(cute.size(x.shape)):+ res[i] = cute.arch.exp2(res[i])+ return res.load()+ else:+ return cute.arch.exp2(x)+++@dsl_user_op+def log2f(a: float | Float32, *, loc=None, ip=None) -> Float32:+ return Float32(+ llvm.inline_asm(+ T.f32(),+ [Float32(a).ir_value(loc=loc, ip=ip)],+ "lg2.approx.ftz.f32 $0, $1;",+ "=f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ )++@dsl_user_op+def logf(a: float | Float32, *, loc=None, ip=None) -> Float32:+ return log2f(a, loc=loc, ip=ip) * math.log(2.0)+++@dsl_user_op+def fmax(+ a: float | Float32, b: float | Float32, c: float | Float32 | None = None, *, loc=None, ip=None+) -> Float32:+ return Float32(+ nvvm.fmax(+ Float32(a).ir_value(loc=loc, ip=ip),+ Float32(b).ir_value(loc=loc, ip=ip),+ c=Float32(c).ir_value(loc=loc, ip=ip) if c is not None else None,+ loc=loc,+ ip=ip,+ )+ )+++def fmax_reduce(+ x: cute.TensorSSA, init_val: float | Float32 | None = None, arch: cutlass.Constexpr[int] = 80+) -> Float32:+ if const_expr(arch < 100 or cute.size(x.shape) % 8 != 0):+ # if const_expr(init_val is None):+ # init_val = -cutlass.Float32.if+ # return x.reduce(cute.ReductionOp.MAX, init_val, 0)+ res = cute.make_rmem_tensor(x.shape, Float32)+ res.store(x)+ # local_max = [res[0], res[1]]+ # for i in cutlass.range_constexpr(2, cute.size(x.shape), 2):+ # local_max[0] = fmax(local_max[0], res[i + 0])+ # local_max[1] = fmax(local_max[1], res[i + 1])+ # local_max[0] = fmax(local_max[0], local_max[1])+ # return local_max[0] if const_expr(init_val is None) else fmax(local_max[0], init_val)+ local_max = [res[0], res[1], res[2], res[3]]+ for i in cutlass.range_constexpr(4, cute.size(x.shape), 4):+ local_max[0] = fmax(local_max[0], res[i + 0])+ local_max[1] = fmax(local_max[1], res[i + 1])+ local_max[2] = fmax(local_max[2], res[i + 2])+ local_max[3] = fmax(local_max[3], res[i + 3])+ local_max[0] = fmax(local_max[0], local_max[1])+ local_max[2] = fmax(local_max[2], local_max[3])+ local_max[0] = fmax(local_max[0], local_max[2])+ return local_max[0] if const_expr(init_val is None) else fmax(local_max[0], init_val)+ else:+ # [2025-06-15] x.reduce only seems to use 50% 3-input max and 50% 2-input max+ # We instead force the 3-input max.+ res = cute.make_rmem_tensor(x.shape, Float32)+ res.store(x)+ local_max_0 = fmax(init_val, res[0], res[1]) if const_expr(init_val is not None) else fmax(res[0], res[1])+ local_max = [+ local_max_0,+ fmax(res[2], res[3]),+ fmax(res[4], res[5]),+ fmax(res[6], res[7]),+ ]+ for i in cutlass.range_constexpr(8, cute.size(x.shape), 8):+ local_max[0] = fmax(local_max[0], res[i], res[i + 1])+ local_max[1] = fmax(local_max[1], res[i + 2], res[i + 3])+ local_max[2] = fmax(local_max[2], res[i + 4], res[i + 5])+ local_max[3] = fmax(local_max[3], res[i + 6], res[i + 7])+ local_max[0] = fmax(local_max[0], local_max[1])+ return fmax(local_max[0], local_max[2], local_max[3])+++def fadd_reduce(+ x: cute.TensorSSA, init_val: float | Float32 | None = None, arch: cutlass.Constexpr[int] = 80+) -> Float32:+ if const_expr(arch < 100 or cute.size(x.shape) % 8 != 0):+ if const_expr(init_val is None):+ init_val = Float32.zero+ return x.reduce(cute.ReductionOp.ADD, init_val, 0)+ # res = cute.make_rmem_tensor(x.shape, Float32)+ # res.store(x)+ # local_sum = [res[0], res[1], res[2], res[3]]+ # for i in cutlass.range_constexpr(4, cute.size(x.shape), 4):+ # local_sum[0] += res[i + 0]+ # local_sum[1] += res[i + 1]+ # local_sum[2] += res[i + 2]+ # local_sum[3] += res[i + 3]+ # local_sum[0] += local_sum[1]+ # local_sum[2] += local_sum[3]+ # local_sum[0] += local_sum[2]+ # return local_sum[0] if const_expr(init_val is None) else local_sum[0] + init_val+ else:+ res = cute.make_rmem_tensor(x.shape, Float32)+ res.store(x)+ local_sum_0 = (+ add_packed_f32x2((init_val, 0.0), (res[0], res[1]))+ # add_packed_f32x2((init_val / 2, init_val / 2), (res[0], res[1]))+ if const_expr(init_val is not None)+ else (res[0], res[1])+ )+ local_sum = [local_sum_0, (res[2], res[3]), (res[4], res[5]), (res[6], res[7])]+ for i in cutlass.range_constexpr(8, cute.size(x.shape), 8):+ local_sum[0] = add_packed_f32x2(local_sum[0], (res[i + 0], res[i + 1]))+ local_sum[1] = add_packed_f32x2(local_sum[1], (res[i + 2], res[i + 3]))+ local_sum[2] = add_packed_f32x2(local_sum[2], (res[i + 4], res[i + 5]))+ local_sum[3] = add_packed_f32x2(local_sum[3], (res[i + 6], res[i + 7]))+ local_sum[0] = add_packed_f32x2(local_sum[0], local_sum[1])+ local_sum[2] = add_packed_f32x2(local_sum[2], local_sum[3])+ local_sum[0] = add_packed_f32x2(local_sum[0], local_sum[2])+ return local_sum[0][0] + local_sum[0][1]+++@dsl_user_op+def atomic_add_fp32(a: float | Float32, gmem_ptr: cute.Pointer, *, loc=None, ip=None) -> None:+ # gmem_ptr_i64 = gmem_ptr.toint(loc=loc, ip=ip).ir_value()+ # # cache_hint = cutlass.Int64(0x12F0000000000000)+ # llvm.inline_asm(+ # None,+ # [gmem_ptr_i64, Float32(a).ir_value(loc=loc, ip=ip)],+ # # [gmem_ptr_i64, Float32(a).ir_value(loc=loc, ip=ip), cache_hint.ir_value()],+ # "red.global.add.f32 [$0], $1;",+ # # "red.global.add.L2::cache_hint.f32 [$0], $1, 0x12F0000000000000;",+ # # "red.global.add.L2::cache_hint.f32 [$0], $1, $2;",+ # "l,f",+ # # "l,f,l",+ # has_side_effects=True,+ # is_align_stack=False,+ # asm_dialect=llvm.AsmDialect.AD_ATT,+ # )+ nvvm.atomicrmw(+ op=nvvm.AtomicOpKind.FADD, ptr=gmem_ptr.llvm_ptr, a=Float32(a).ir_value()+ )+++@dsl_user_op+def elem_pointer(x: cute.Tensor, coord: cute.Coord, *, loc=None, ip=None) -> cute.Pointer:+ return x.iterator + cute.crd2idx(coord, x.layout, loc=loc, ip=ip)+++@dsl_user_op+def elem_pointer_i64(x: cute.Tensor, coord: cute.Coord, *, loc=None, ip=None) -> cute.Pointer:+ flat_coord_i64 = tuple(cutlass.Int64(c) for c in cute.flatten(coord))+ flat_stride = cute.flatten_to_tuple(x.stride)+ assert len(flat_coord_i64) == len(flat_stride), "Coordinate and stride must have the same length"+ offset = sum(c * s for c, s in zip(flat_coord_i64, flat_stride))+ # HACK: we assume that applying the offset does not change the pointer alignment+ byte_offset = offset * x.element_type.width // 8+ return cute.make_ptr(+ x.element_type,+ x.iterator.toint() + byte_offset,+ x.memspace,+ assumed_align=x.iterator.alignment,+ )+++def predicate_k(tAcA: cute.Tensor, limit: cutlass.Int32) -> cute.Tensor:+ # Only compute predicates for the "k" dimension. For the mn dimension, we will use "if"+ tApA = cute.make_rmem_tensor(+ cute.make_layout(+ (cute.size(tAcA, mode=[0, 1]), cute.size(tAcA, mode=[1]), cute.size(tAcA, mode=[2])),+ stride=(cute.size(tAcA, mode=[2]), 0, 1),+ ),+ cutlass.Boolean,+ )+ for rest_v in cutlass.range_constexpr(tApA.shape[0]):+ for rest_k in cutlass.range_constexpr(tApA.shape[2]):+ tApA[rest_v, 0, rest_k] = cute.elem_less(tAcA[(0, rest_v), 0, rest_k][1], limit)+ return tApA+++def canonical_warp_group_idx(sync: bool = True) -> cutlass.Int32:+ warp_group_idx = cute.arch.thread_idx()[0] // 128+ if const_expr(sync):+ warp_group_idx = cute.arch.make_warp_uniform(warp_group_idx)+ return warp_group_idx+++# @dsl_user_op+# def warp_vote_any_lt(a: float | Float32, b: float | Float32, *, loc=None, ip=None) -> cutlass.Boolean:+# mask = cutlass.Int32(-1)+# return cutlass.Boolean(+# llvm.inline_asm(+# T.i32(),+# [Float32(a).ir_value(loc=loc, ip=ip), Float32(b).ir_value(loc=loc, ip=ip), mask.ir_value(loc=loc, ip=ip)],+# ".pred p1, p2;\n"+# "setp.lt.f32 p1, $1, $2;\n"+# "vote.sync.any.pred p2, p1, $3;\n"+# "selp.u32 $0, 1, 0, p2;",+# # "selp.u32 $0, 1, 0, p1;",+# "=r,f,f,r",+# has_side_effects=False,+# is_align_stack=False,+# asm_dialect=llvm.AsmDialect.AD_ATT,+# )+# )+++def shuffle_sync(+ value: cute.Numeric,+ offset: cute.typing.Int,+ width: cutlass.Constexpr[int] = cute.arch.WARP_SIZE,+) -> cute.Numeric:+ assert value.width % 32 == 0, "value type must be a multiple of 32 bits"+ # 1 -> 0b11111, 2 -> 0b11110, 4 -> 0b11100, 8 -> 0b11000, 16 -> 0b10000, 32 -> 0b00000+ mask = cute.arch.WARP_SIZE - width+ clamp = cute.arch.WARP_SIZE - 1+ mask_and_clamp = mask << 8 | clamp+ val = cute.make_rmem_tensor(1, type(value))+ val[0] = value+ val_i32 = cute.recast_tensor(val, cutlass.Int32)+ for i in cutlass.range_constexpr(cute.size(val_i32)):+ val_i32[i] = cute.arch.shuffle_sync(val_i32[i], offset, mask_and_clamp=mask_and_clamp)+ return val[0]+++@dsl_user_op+def barrier_or(+ value: Int32, barrier_id: int, number_of_threads: int, *, loc=None, ip=None+) -> Int32:+ """Named barrier that also returns, to every participant, whether any of+ them passed a nonzero ``value``. All participants must call it together,+ as for ``cute.arch.barrier``."""+ return Int32(+ llvm.inline_asm(+ T.i32(),+ [Int32(value).ir_value(loc=loc, ip=ip)],+ "{\n.reg .pred p, q;\n"+ "setp.ne.s32 p, $1, 0;\n"+ f"bar.red.or.pred q, {barrier_id}, {number_of_threads}, p;\n"+ "selp.s32 $0, 1, 0, q;\n}",+ "=r,r",+ has_side_effects=True,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ )+++@dsl_user_op+def shr_u32(val: cutlass.Uint32, shift: cutlass.Uint32, *, loc=None, ip=None) -> cutlass.Uint32:+ return cutlass.Uint32(+ llvm.inline_asm(+ T.i32(),+ [cutlass.Uint32(val).ir_value(loc=loc, ip=ip), cutlass.Uint32(shift).ir_value(loc=loc, ip=ip)],+ "shr.s32 $0, $1, $2;",+ "=r,r,r",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ )+++def warp_prefix_sum(val: cutlass.Int32, lane: Optional[cutlass.Int32] = None) -> cutlass.Int32:+ if const_expr(lane is None):+ lane = cute.arch.lane_idx()+ # if cute.arch.thread_idx()[0] >= 128 and cute.arch.thread_idx()[0] < 128 + 32 and cute.arch.block_idx()[0] == 0: cute.printf("tidx = %d, val = %d", cute.arch.thread_idx()[0] % 32, val)+ for i in cutlass.range_constexpr(int(math.log2(cute.arch.WARP_SIZE))):+ offset = 1 << i+ # Very important that we set mask_and_clamp to 0+ partial_sum = cute.arch.shuffle_sync_up(val, offset=offset, mask_and_clamp=0)+ if lane >= offset:+ val += partial_sum+ # if cute.arch.thread_idx()[0] >= 128 and cute.arch.thread_idx()[0] < 128 + 32 and cute.arch.block_idx()[0] == 0: cute.printf("tidx = %d, partial_sum = %d, val = %d", cute.arch.thread_idx()[0] % 32, partial_sum, val)+ return val+++@dsl_user_op+def cvt_f16x2_f32(a: float | Float32, b: float | Float32, to_dtype: Type, *, loc=None, ip=None) -> cutlass.Int32:+ assert to_dtype in [cutlass.BFloat16, cutlass.Float16], "to_dtype must be BFloat16 or Float16"+ return cutlass.Int32(+ llvm.inline_asm(+ T.i32(),+ [Float32(a).ir_value(loc=loc, ip=ip), Float32(b).ir_value(loc=loc, ip=ip)],+ f"cvt.rn.{'bf16x2' if to_dtype is cutlass.BFloat16 else 'f16x2'}.f32 $0, $2, $1;",+ "=r,f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ )+++@overload+def cvt_f16(src: cute.Tensor, dst: cute.Tensor) -> None: ...++@overload+def cvt_f16(src: cute.Tensor, dtype: Type[cute.Numeric]) -> cute.Tensor: ...++def cvt_f16(src: cute.Tensor, dst_or_dtype):+ """Convert Float32 tensor to Float16/BFloat16.++ Args:+ src: Source tensor with Float32 element type+ dst_or_dtype: Either a destination tensor or a dtype (Float16/BFloat16)++ Returns:+ None if dst is a tensor, or a new tensor if dtype is provided+ """+ if const_expr(isinstance(dst_or_dtype, type)):+ # dtype variant: create new tensor and call the tensor variant+ dtype = dst_or_dtype+ dst = cute.make_rmem_tensor(src.shape, dtype)+ cvt_f16(src, dst)+ return dst+ else:+ # tensor variant: write to dst+ dst = dst_or_dtype+ assert cute.size(dst.shape) == cute.size(src.shape), "dst and src must have the same size"+ assert cute.size(src.shape) % 2 == 0, "src must have an even number of elements"+ assert dst.element_type in [cutlass.BFloat16, cutlass.Float16], "dst must be BFloat16 or Float16"+ assert src.element_type is Float32, "src must be Float32"+ dst_i32 = cute.recast_tensor(dst, cutlass.Int32)+ assert cute.size(dst_i32.shape) * 2 == cute.size(src.shape)+ for i in cutlass.range_constexpr(cute.size(dst_i32)):+ dst_i32[i] = cvt_f16x2_f32(src[2 * i], src[2 * i + 1], dst.element_type)+++@dsl_user_op+def evaluate_polynomial(x: Float32, poly: Tuple[Float32, ...], *, loc=None, ip=None) -> Float32:+ deg = len(poly) - 1+ out = poly[deg]+ for i in cutlass.range_constexpr(deg - 1, -1, -1):+ out = out * x + poly[i]+ return out+++@dsl_user_op+def evaluate_polynomial_2(x: Float32, y: Float32, poly: Tuple[Float32, ...], *, loc=None, ip=None) -> Tuple[Float32, Float32]:+ deg = len(poly) - 1+ out = (poly[deg], poly[deg])+ for i in cutlass.range_constexpr(deg - 1, -1, -1):+ out = fma_packed_f32x2(out, (x, y), (poly[i], poly[i]))+ return out+++@dsl_user_op+def add_round_down(x: float | Float32, y: float | Float32, *, loc=None, ip=None) -> Float32:+ # There's probably a way to call llvm or nvvm to do this instead of ptx+ return cutlass.Float32(+ llvm.inline_asm(+ T.f32(),+ [Float32(x).ir_value(loc=loc, ip=ip), Float32(y).ir_value(loc=loc, ip=ip)],+ f"add.rm.ftz.f32 $0, $1, $2;",+ "=f,f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ )+++@dsl_user_op+def combine_int_frac_ex2(x_rounded: Float32, frac_ex2: Float32, *, loc=None, ip=None) -> Float32:+ return cutlass.Float32(+ llvm.inline_asm(+ T.f32(),+ [Float32(x_rounded).ir_value(loc=loc, ip=ip), Float32(frac_ex2).ir_value(loc=loc, ip=ip)],+ "{\n\t"+ ".reg .s32 x_rounded_i, frac_ex_i, x_rounded_e, out_i;\n\t"+ "mov.b32 x_rounded_i, $1;\n\t"+ "mov.b32 frac_ex_i, $2;\n\t"+ "shl.b32 x_rounded_e, x_rounded_i, 23;\n\t"+ # add.u32 generates IMAD instruction and add.s32 generates LEA instruction+ # IMAD uses the FMA pipeline and LEA uses the ALU pipeline, afaik+ "add.s32 out_i, x_rounded_e, frac_ex_i;\n\t"+ "mov.b32 $0, out_i;\n\t"+ "}\n",+ "=f,f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ )+++@dsl_user_op+def ex2_emulation(x: Float32, *, loc=None, ip=None) -> Float32:+ # We assume x <= 127.0+ poly_ex2_deg3 = (1.0, 0.695146143436431884765625, 0.227564394474029541015625, 0.077119089663028717041015625)+ fp32_round_int = float(2**23 + 2**22)+ x_clamped = cute.arch.fmax(x, -127.0)+ # We want to round down here, so that the fractional part is in [0, 1)+ x_rounded = add_round_down(x_clamped, fp32_round_int, loc=loc, ip=ip)+ # The integer floor of x is now in the last 8 bits of x_rounded+ # We assume the next 2 ops round to nearest even. The rounding mode is important.+ x_rounded_back = x_rounded - fp32_round_int+ x_frac = x_clamped - x_rounded_back+ x_frac_ex2 = evaluate_polynomial(x_frac, poly_ex2_deg3, loc=loc, ip=ip)+ return combine_int_frac_ex2(x_rounded, x_frac_ex2, loc=loc, ip=ip)+++# TODO: check that the ex2_emulation_2 produces the same SASS as the ptx version+@dsl_user_op+def ex2_emulation_2(x: Float32, y: Float32, *, loc=None, ip=None) -> Tuple[Float32, Float32]:+ # We assume x <= 127.0 and y <= 127.0+ poly_ex2_deg3 = (1.0, 0.695146143436431884765625, 0.227564394474029541015625, 0.077119089663028717041015625)+ fp32_round_int = float(2**23 + 2**22)+ xy_clamped = (cute.arch.fmax(x, -127.0), cute.arch.fmax(y, -127.0))+ # We want to round down here, so that the fractional part is in [0, 1)+ xy_rounded = cute.arch.add_packed_f32x2(xy_clamped, (fp32_round_int, fp32_round_int), rnd="rm")+ # The integer floor of x & y are now in the last 8 bits of xy_rounded+ # We want the next 2 ops to round to nearest even. The rounding mode is important.+ xy_rounded_back = sub_packed_f32x2(xy_rounded, (fp32_round_int, fp32_round_int))+ xy_frac = sub_packed_f32x2(xy_clamped, xy_rounded_back)+ xy_frac_ex2 = evaluate_polynomial_2(*xy_frac, poly_ex2_deg3, loc=loc, ip=ip)+ x_out = combine_int_frac_ex2(xy_rounded[0], xy_frac_ex2[0], loc=loc, ip=ip)+ y_out = combine_int_frac_ex2(xy_rounded[1], xy_frac_ex2[1], loc=loc, ip=ip)+ return x_out, y_out+++@dsl_user_op+def e2e_asm2(x: Float32, y: Float32, *, loc=None, ip=None) -> Tuple[Float32, Float32]:+ out_f32x2 = llvm.inline_asm(+ llvm.StructType.get_literal([T.f32(), T.f32()]),+ [Float32(x).ir_value(loc=loc, ip=ip), Float32(y, loc=loc, ip=ip).ir_value()],+ "{\n\t"+ ".reg .f32 f1, f2, f3, f4, f5, f6, f7;\n\t"+ ".reg .b64 l1, l2, l3, l4, l5, l6, l7, l8, l9, l10;\n\t"+ ".reg .s32 r1, r2, r3, r4, r5, r6, r7, r8;\n\t"+ "max.ftz.f32 f1, $2, 0fC2FE0000;\n\t"+ "max.ftz.f32 f2, $3, 0fC2FE0000;\n\t"+ "mov.b64 l1, {f1, f2};\n\t"+ "mov.f32 f3, 0f4B400000;\n\t"+ "mov.b64 l2, {f3, f3};\n\t"+ "add.rm.ftz.f32x2 l7, l1, l2;\n\t"+ "sub.rn.ftz.f32x2 l8, l7, l2;\n\t"+ "sub.rn.ftz.f32x2 l9, l1, l8;\n\t"+ "mov.f32 f7, 0f3D9DF09D;\n\t"+ "mov.b64 l6, {f7, f7};\n\t"+ "mov.f32 f6, 0f3E6906A4;\n\t"+ "mov.b64 l5, {f6, f6};\n\t"+ "mov.f32 f5, 0f3F31F519;\n\t"+ "mov.b64 l4, {f5, f5};\n\t"+ "mov.f32 f4, 0f3F800000;\n\t"+ "mov.b64 l3, {f4, f4};\n\t"+ "fma.rn.ftz.f32x2 l10, l9, l6, l5;\n\t"+ "fma.rn.ftz.f32x2 l10, l10, l9, l4;\n\t"+ "fma.rn.ftz.f32x2 l10, l10, l9, l3;\n\t"+ "mov.b64 {r1, r2}, l7;\n\t"+ "mov.b64 {r3, r4}, l10;\n\t"+ "shl.b32 r5, r1, 23;\n\t"+ "add.s32 r7, r5, r3;\n\t"+ "shl.b32 r6, r2, 23;\n\t"+ "add.s32 r8, r6, r4;\n\t"+ "mov.b32 $0, r7;\n\t"+ "mov.b32 $1, r8;\n\t"+ "}\n",+ "=r,=r,f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ out0 = Float32(llvm.extractvalue(T.f32(), out_f32x2, [0], loc=loc, ip=ip))+ out1 = Float32(llvm.extractvalue(T.f32(), out_f32x2, [1], loc=loc, ip=ip))+ return out0, out1+@dsl_user_op+def domain_offset_aligned(coord: cute.Coord, tensor: cute.Tensor, *, loc=None, ip=None) -> cute.Tensor:+ assert isinstance(tensor.iterator, cute.Pointer)+ # We assume that applying the offset does not change the pointer alignment+ new_ptr = cute.make_ptr(+ tensor.element_type,+ elem_pointer(tensor, coord).toint(),+ tensor.memspace,+ assumed_align=tensor.iterator.alignment,+ )+ return cute.make_tensor(new_ptr, tensor.layout)+++@dsl_user_op+def domain_offset_i64(coord: cute.Coord, tensor: cute.Tensor, *, loc=None, ip=None) -> cute.Tensor:+ flat_coord_i64 = tuple(cutlass.Int64(c) for c in cute.flatten(coord))+ flat_stride = cute.flatten_to_tuple(tensor.stride)+ assert len(flat_coord_i64) == len(+ flat_stride+ ), "Coordinate and stride must have the same length"+ offset = sum(c * s for c, s in zip(flat_coord_i64, flat_stride))+ assert isinstance(tensor.iterator, cute.Pointer)+ # HACK: we assume that applying the offset does not change the pointer alignment+ new_ptr = cute.make_ptr(+ tensor.element_type,+ tensor.iterator.toint() + offset * tensor.element_type.width // 8,+ tensor.memspace,+ assumed_align=tensor.iterator.max_alignment,+ )+ return cute.make_tensor(new_ptr, tensor.layout)+++@dsl_user_op+def coord_offset_i64(+ tensor: cute.Tensor, idx: cute.typing.Int, dim: int, *, loc=None, ip=None+) -> cute.Tensor:+ offset = cutlass.Int64(idx) * cute.size(tensor.stride[dim])+ assert isinstance(tensor.iterator, cute.Pointer)+ # HACK: we assume that applying the offset does not change the pointer alignment+ new_ptr = cute.make_ptr(+ tensor.element_type,+ tensor.iterator.toint() + offset * tensor.element_type.width // 8,+ tensor.memspace,+ assumed_align=tensor.iterator.max_alignment,+ )+ new_layout = cute.slice_(tensor.layout, (*[None] * dim, 0, *[None] * (cute.rank(tensor) - dim - 1)))+ return cute.make_tensor(new_ptr, new_layout)+++def scalar_to_ssa(a: cute.Numeric, dtype) -> cute.TensorSSA:+ """ Convert a scalar to a cute TensorSSA of shape (1,) and given dtype """+ vec = cute.make_rmem_tensor(1, dtype)+ vec[0] = a+ return vec.load()+++def ssa_to_scalar(val):+ """ Could inline but nice for reflecting the above api """+ return val[0]\ No newline at end of filediff --git a/src/nvfp4_decode_kernel/_kernel.py b/src/nvfp4_decode_kernel/_kernel.pynew file mode 100644index 0000000..e2d6365--- /dev/null+++ b/src/nvfp4_decode_kernel/_kernel.py@@ -0,0 +1,416 @@+"""Composition layer between the public API and private kernel operations."""++from __future__ import annotations++import torch++from .interface import RESIDUAL_ROW_TILE, DecodeTailWrite+from .scale_mode import DEFAULT_SCALE_MODE, scale_format, sf_vec_size+++def _padded_query_buffer(+ query: torch.Tensor,+ rows: int,+ scratch: torch.Tensor | None,+) -> torch.Tensor:+ """The BF16 query padded to the residual MMA's row tile.++ Deliberately not cached here. A framework that sizes its KV cache from the+ memory left over after a profiling run — vLLM does — would find a buffer+ grown on a later call coming out of memory already promised to the cache.+ Leaving the allocation to the caller lets it happen during profiling, at+ the price of the contract ``fp4_decode`` documents: zeroed once, because+ the quantizer writes ``[row, 0, head, :]`` and nothing writes the rest.+ """+ heads, head_dim = query.shape[1], query.shape[2]+ if scratch is None:+ return torch.zeros(+ rows,+ RESIDUAL_ROW_TILE,+ heads,+ head_dim,+ dtype=torch.bfloat16,+ device=query.device,+ )+ if (+ scratch.dtype is not torch.bfloat16+ or scratch.device != query.device+ or scratch.ndim != 4+ or scratch.shape[0] < rows+ or tuple(scratch.shape[1:]) != (RESIDUAL_ROW_TILE, heads, head_dim)+ or not scratch.is_contiguous()+ ):+ raise ValueError(+ "query_padded_scratch must be a contiguous BF16 tensor on the "+ f"query's device shaped [at least {rows}, {RESIDUAL_ROW_TILE}, "+ f"{heads}, {head_dim}]"+ )+ return scratch[:rows]+++def _check_quantizer_scratch(+ scratch: torch.Tensor | None,+ name: str,+ rows: int,+ trailing: tuple[int, ...],+ device: torch.device,+) -> None:+ """Vet a caller-owned buffer for one of the quantizer's two outputs.++ Same bargain as ``_padded_query_buffer``, and made for the same reason: a+ framework that sizes its KV cache around what profiling left free wants+ every decode buffer to exist before that measurement, and wants the step+ itself to neither allocate nor fill. Only the row count may exceed what+ this batch needs, since rows are dim 0 and everything the decode kernel+ checks about the scale layout is an inner stride.++ Both buffers reach kernels compiled with ``assumed_align=16``, so the+ requirement is really on the address rather than on the layout. Demanding+ storage offset zero buys that without an alignment test: offset zero is+ the start of a PyTorch allocation, which the caching allocator hands out+ aligned to 512 bytes, far past the 16 the kernels assume. A contiguous+ view onto the middle of somebody else's tensor carries no such guarantee.+ """+ if scratch is None:+ return+ if (+ scratch.dtype is not torch.uint8+ or scratch.device != device+ or scratch.ndim != 1 + len(trailing)+ or scratch.shape[0] < rows+ or tuple(scratch.shape[1:]) != trailing+ or not scratch.is_contiguous()+ or scratch.storage_offset() != 0+ ):+ extents = ", ".join(str(extent) for extent in trailing)+ raise ValueError(+ f"{name} must be a contiguous UINT8 tensor on the query's device "+ f"shaped [at least {rows}, {extents}], starting at storage "+ "offset zero"+ )+++def fp4_decode_impl(+ query: torch.Tensor | None,+ key_pages_fp4: torch.Tensor,+ key_scales: torch.Tensor,+ value_pages_fp4: torch.Tensor,+ value_scales: torch.Tensor,+ fp4_page_table: torch.Tensor,+ seqused_fp4: torch.Tensor,+ *,+ query_fp4: torch.Tensor | None = None,+ query_scales: torch.Tensor | None = None,+ query_fp4_second: torch.Tensor | None = None,+ query_scales_second: torch.Tensor | None = None,+ residual_key_pages_bf16: torch.Tensor | None = None,+ residual_value_pages_bf16: torch.Tensor | None = None,+ residual_page_ids: torch.Tensor | None = None,+ seqused_residual: torch.Tensor | None = None,+ seqused_sink: torch.Tensor | None = None,+ has_bf16: torch.Tensor | None = None,+ softmax_scale: float | None = None,+ query_row_indices: torch.Tensor | None = None,+ out: torch.Tensor | None = None,+ out_indices: torch.Tensor | None = None,+ trusted_metadata: bool = False,+ num_splits: int = 1,+ query_padded_scratch: torch.Tensor | None = None,+ query_fp4_scratch: torch.Tensor | None = None,+ query_scales_scratch: torch.Tensor | None = None,+ query_fp4_second_scratch: torch.Tensor | None = None,+ query_scales_second_scratch: torch.Tensor | None = None,+ split_partial_scratch: torch.Tensor | None = None,+ split_lse_scratch: torch.Tensor | None = None,+ query_terms: int = 1,+ scale_mode: str = DEFAULT_SCALE_MODE,+ query_rotation: str = "none",+ tail_write: DecodeTailWrite | None = None,+) -> torch.Tensor:+ """Prepare either query contract and run the shared decode core."""+ from ._decode import decode_fp4, decode_fp4_split+ from ._quantize import quantize_query++ # Any count partitions the pages (ceil division per split), and the+ # combine compiles one kernel per power-of-two tier above it. Checked+ # before any quantization so a bad value costs nothing.+ if not isinstance(num_splits, int) or num_splits < 1:+ raise ValueError(+ f"num_splits must be a positive integer, got {num_splits!r}"+ )+ if query_terms not in (1, 2):+ raise ValueError(f"query_terms must be 1 or 2, got {query_terms!r}")+ fmt = scale_format(scale_mode)++ has_bf16_query = query is not None+ has_fp4_query = any(+ tensor is not None+ for tensor in (+ query_fp4,+ query_scales,+ query_fp4_second,+ query_scales_second,+ )+ )+ if has_bf16_query == has_fp4_query:+ raise ValueError(+ "supply either BF16 query or query_fp4 + query_scales"+ )++ if has_bf16_query:+ assert query is not None+ if any(+ tensor is not None+ for tensor in (+ query_fp4,+ query_scales,+ query_fp4_second,+ query_scales_second,+ )+ ):+ raise ValueError(+ "BF16 query is mutually exclusive with pre-quantized query "+ "terms"+ )+ if softmax_scale is None:+ softmax_scale = query.shape[-1] ** -0.5+ rows = (+ query.shape[0]+ if query_row_indices is None+ else query_row_indices.shape[0]+ )+ query_padded_bf16 = (+ _padded_query_buffer(query, rows, query_padded_scratch)+ if residual_key_pages_bf16 is not None+ else None+ )+ heads, head_dim = query.shape[1], query.shape[2]+ _check_quantizer_scratch(+ query_fp4_scratch,+ "query_fp4_scratch",+ rows,+ (1, heads, head_dim // 2),+ query.device,+ )+ _check_quantizer_scratch(+ query_scales_scratch,+ "query_scales_scratch",+ rows,+ (1, heads, head_dim // (4 * sf_vec_size(scale_mode)), 32, 4, 4),+ query.device,+ )+ if query_terms == 1 and (+ query_fp4_second_scratch is not None+ or query_scales_second_scratch is not None+ ):+ raise ValueError(+ "second query scratch requires query_terms=2"+ )+ if query_terms == 2:+ _check_quantizer_scratch(+ query_fp4_second_scratch,+ "query_fp4_second_scratch",+ rows,+ (1, heads, head_dim // 2),+ query.device,+ )+ _check_quantizer_scratch(+ query_scales_second_scratch,+ "query_scales_second_scratch",+ rows,+ (1, heads, head_dim // (4 * sf_vec_size(scale_mode)), 32, 4, 4),+ query.device,+ )+ quantized_query = quantize_query(+ query,+ row_indices=query_row_indices,+ query_padded_out=query_padded_bf16,+ heads_kv=key_pages_fp4.shape[2],+ query_fp4_scratch=query_fp4_scratch,+ query_scales_scratch=query_scales_scratch,+ query_fp4_second_scratch=query_fp4_second_scratch,+ query_scales_second_scratch=query_scales_second_scratch,+ query_terms=query_terms,+ scale_mode=scale_mode,+ rotation=query_rotation,+ tail_write=tail_write,+ )+ if query_terms == 2:+ (+ query_fp4,+ query_scales,+ query_fp4_second,+ query_scales_second,+ ) = quantized_query+ else:+ query_fp4, query_scales = quantized_query+ query_fp4_second = None+ query_scales_second = None+ else:+ if query_fp4 is None or query_scales is None:+ raise ValueError(+ "query_fp4 and query_scales must be provided together"+ )+ has_second_pair = (+ query_fp4_second is not None+ and query_scales_second is not None+ )+ if (query_fp4_second is None) != (query_scales_second is None):+ raise ValueError(+ "query_fp4_second and query_scales_second must be provided "+ "together"+ )+ if query_terms == 2 and not has_second_pair:+ raise ValueError(+ "query_terms=2 needs query_fp4_second and "+ "query_scales_second"+ )+ if query_terms == 1 and has_second_pair:+ raise ValueError(+ "second pre-quantized query pair requires query_terms=2"+ )+ if query_row_indices is not None:+ raise ValueError(+ "query_row_indices applies only to the BF16 query path"+ )+ for name, scratch in (+ ("query_padded_scratch", query_padded_scratch),+ ("query_fp4_scratch", query_fp4_scratch),+ ("query_scales_scratch", query_scales_scratch),+ ("query_fp4_second_scratch", query_fp4_second_scratch),+ (+ "query_scales_second_scratch",+ query_scales_second_scratch,+ ),+ ):+ if scratch is not None:+ raise ValueError(+ f"{name} applies only to the BF16 query path"+ )+ if scale_mode != DEFAULT_SCALE_MODE:+ # The caller already chose one when they quantized the query+ # themselves, and this argument would name a second.+ raise ValueError("scale_mode applies only to the BF16 query path")+ if query_rotation != "none":+ raise ValueError(+ "query_rotation applies only to the BF16 query path"+ )+ if tail_write is not None:+ raise ValueError("tail_write applies only to the BF16 query path")+ if softmax_scale is None:+ softmax_scale = 128**-0.5+ query_padded_bf16 = None++ # A residual reaches split-K only when every one of its arguments is+ # present. Keying off a single argument would let a partially supplied+ # residual be dropped there instead of raising the all-or-none error that+ # the non-split path enforces.+ residual_arguments = (+ residual_key_pages_bf16,+ residual_value_pages_bf16,+ residual_page_ids,+ seqused_residual,+ )+ complete_residual = all(+ argument is not None for argument in residual_arguments+ )+ splittable = (+ all(argument is None for argument in residual_arguments)+ and has_bf16 is None+ ) or (complete_residual and query_padded_bf16 is not None)+ # Only a scatter by index forces the single-tile path; the split path can+ # write into a caller's buffer, it just cannot reorder rows on the way.+ scatter_by_index = out_indices is not None+ # A split the dispatcher cannot serve is refused rather than served with+ # one tile, so that a caller asking for the split path either gets it or+ # hears why not.+ if num_splits > 1 and not splittable:+ raise ValueError(+ "num_splits > 1 needs either no residual argument at all or a "+ "complete residual on the BF16 query path; this combination "+ "runs on the single-tile path only"+ )+ if num_splits > 1 and scatter_by_index:+ raise ValueError(+ "num_splits > 1 cannot be combined with out_indices, because the "+ "split path's combine writes the batch's rows in order"+ )+ if num_splits > 1:+ # Keep the single public entry's full contract checks when dispatching+ # to the split implementation. Trusted metadata skips only the+ # intentional device-value scans, not host shape/dtype validation.+ decode_fp4(+ query_fp4=query_fp4,+ query_scales=query_scales,+ query_fp4_second=query_fp4_second,+ query_scales_second=query_scales_second,+ query_padded_bf16=query_padded_bf16,+ key_pages_fp4=key_pages_fp4,+ key_scales=key_scales,+ value_pages_fp4=value_pages_fp4,+ value_scales=value_scales,+ fp4_page_table=fp4_page_table,+ seqused_fp4=seqused_fp4,+ residual_key_pages_bf16=residual_key_pages_bf16,+ residual_value_pages_bf16=residual_value_pages_bf16,+ residual_page_ids=residual_page_ids,+ seqused_residual=seqused_residual,+ seqused_sink=seqused_sink,+ has_bf16=has_bf16,+ softmax_scale=softmax_scale,+ out=out,+ scale_format=fmt,+ out_indices=out_indices,+ trusted_metadata=trusted_metadata,+ validate_only=True,+ )+ return decode_fp4_split(+ query_fp4=query_fp4,+ query_scales=query_scales,+ query_fp4_second=query_fp4_second,+ query_scales_second=query_scales_second,+ key_pages_fp4=key_pages_fp4,+ key_scales=key_scales,+ value_pages_fp4=value_pages_fp4,+ value_scales=value_scales,+ fp4_page_table=fp4_page_table,+ seqused_fp4=seqused_fp4,+ softmax_scale=softmax_scale,+ num_splits=num_splits,+ query_padded_bf16=query_padded_bf16,+ residual_key_pages_bf16=residual_key_pages_bf16,+ residual_value_pages_bf16=residual_value_pages_bf16,+ residual_page_ids=residual_page_ids,+ seqused_residual=seqused_residual,+ seqused_sink=seqused_sink,+ has_bf16=has_bf16,+ out=out,+ split_partial_scratch=split_partial_scratch,+ split_lse_scratch=split_lse_scratch,+ scale_format=fmt,+ )++ return decode_fp4(+ query_fp4=query_fp4,+ query_scales=query_scales,+ query_fp4_second=query_fp4_second,+ query_scales_second=query_scales_second,+ query_padded_bf16=query_padded_bf16,+ key_pages_fp4=key_pages_fp4,+ key_scales=key_scales,+ value_pages_fp4=value_pages_fp4,+ value_scales=value_scales,+ fp4_page_table=fp4_page_table,+ seqused_fp4=seqused_fp4,+ residual_key_pages_bf16=residual_key_pages_bf16,+ residual_value_pages_bf16=residual_value_pages_bf16,+ residual_page_ids=residual_page_ids,+ seqused_residual=seqused_residual,+ seqused_sink=seqused_sink,+ has_bf16=has_bf16,+ softmax_scale=softmax_scale,+ out=out,+ scale_format=fmt,+ out_indices=out_indices,+ trusted_metadata=trusted_metadata,+ )diff --git a/src/nvfp4_decode_kernel/_quantize.py b/src/nvfp4_decode_kernel/_quantize.pynew file mode 100644index 0000000..74f70f6--- /dev/null+++ b/src/nvfp4_decode_kernel/_quantize.py@@ -0,0 +1,205 @@+"""NVFP4 query quantization."""++from __future__ import annotations++import torch++from .interface import DecodeTailWrite+from .quantize_kv_kernel import quantize_key_pages, quantize_value_pages+from .quantize_q_kernel import quantize_decode_q_to_padded_fp4+from .scale_mode import DEFAULT_SCALE_MODE, sf_vec_size+++PAGE_SIZE = 128+++def quantize_query(+ query: torch.Tensor,+ *,+ row_indices: torch.Tensor | None = None,+ query_padded_out: torch.Tensor | None = None,+ heads_kv: int | None = None,+ query_fp4_scratch: torch.Tensor | None = None,+ query_scales_scratch: torch.Tensor | None = None,+ query_fp4_second_scratch: torch.Tensor | None = None,+ query_scales_second_scratch: torch.Tensor | None = None,+ query_terms: int = 1,+ scale_mode: str = DEFAULT_SCALE_MODE,+ rotation: str = "none",+ tail_write: DecodeTailWrite | None = None,+) -> (+ tuple[torch.Tensor, torch.Tensor]+ | tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]+):+ """Quantize BF16 decode rows with the CuTeDSL query kernel.++ Outputs may come from the caller instead of a fresh allocation, and+ may be wider than this batch. Rows are dim 0 of each, so the prefix slice+ taken here leaves every inner stride alone and the scale view still comes+ out with the layout the decode kernel insists on. A supplied scale buffer+ must arrive zeroed: the layout reserves 128 scale slots per KV head and+ the kernel writes only the ``heads // heads_kv`` of them that carry a+ query head. Nothing clears the rest, so a buffer that starts out zeroed+ is byte-for-byte what a fresh allocation would have produced, and one+ that does not stays wrong for as long as it is reused. Which slots those+ are follows ``heads // heads_kv`` while the storage shape does not, so a+ scale buffer also belongs to one ``(heads, heads_kv, head_dim)`` triple+ for its whole life: carrying it across a ``heads_kv`` change leaves the+ previous geometry's scales sitting in slots the new one needs zeroed.+ With ``query_terms=2``, the extra pair encodes the residual left by the+ first pair and the function returns ``(q0, s0, q1, s1)``. The default+ remains the original two-value return. ``rotation="hadamard"`` rotates+ each head first, and ``tail_write`` also places each row's new K/V token+ in the BF16 tail; see :func:`quantize_decode_q_to_padded_fp4`.+ """+ if query_terms not in (1, 2):+ raise ValueError(f"query_terms must be 1 or 2, got {query_terms!r}")+ if query_terms == 1 and (+ query_fp4_second_scratch is not None+ or query_scales_second_scratch is not None+ ):+ raise ValueError(+ "second query scratch requires query_terms=2"+ )+ if query.dtype is not torch.bfloat16 or not query.is_cuda:+ raise ValueError("query must be a BF16 CUDA tensor")+ if query.ndim != 3 or query.stride(-1) != 1:+ raise ValueError("query must have shape [rows, heads, head_dim]")+ if row_indices is not None and (+ row_indices.dtype is not torch.int32+ or not row_indices.is_cuda+ or not row_indices.is_contiguous()+ or row_indices.ndim != 1+ ):+ raise ValueError(+ "row_indices must be contiguous 1-D INT32 CUDA"+ )++ _, heads, head_dim = query.shape+ heads_kv = heads if heads_kv is None else heads_kv+ if heads_kv < 1 or heads % heads_kv:+ raise ValueError("query heads must be divisible by heads_kv")+ rows = query.shape[0] if row_indices is None else row_indices.shape[0]+ atom_k = 4 * sf_vec_size(scale_mode)+ if head_dim % atom_k != 0:+ raise ValueError(f"query head_dim must be divisible by {atom_k}")++ query_fp4 = (+ torch.zeros(+ rows,+ 1,+ heads,+ head_dim // 2,+ dtype=torch.uint8,+ device=query.device,+ )+ if query_fp4_scratch is None+ else query_fp4_scratch[:rows]+ ).view(torch.float4_e2m1fn_x2)++ rest_k = head_dim // atom_k+ scale_storage = (+ torch.zeros(+ rows,+ 1,+ heads,+ rest_k,+ 32,+ 4,+ 4,+ dtype=torch.uint8,+ device=query.device,+ )+ if query_scales_scratch is None+ else query_scales_scratch[:rows]+ )+ native_scales = scale_storage.permute(+ 0, 2, 1, 3, 4, 5, 6+ ).permute(+ 4, 5, 2, 6, 3, 1, 0+ )+ query_scales = native_scales.as_strided(+ (+ 32,+ 4,+ 1,+ 4,+ rest_k,+ heads_kv,+ rows,+ ),+ native_scales.stride(),+ )++ query_fp4_second = None+ query_scales_second = None+ if query_terms == 2:+ query_fp4_second = (+ torch.zeros(+ rows,+ 1,+ heads,+ head_dim // 2,+ dtype=torch.uint8,+ device=query.device,+ )+ if query_fp4_second_scratch is None+ else query_fp4_second_scratch[:rows]+ ).view(torch.float4_e2m1fn_x2)+ scale_storage_second = (+ torch.zeros(+ rows,+ 1,+ heads,+ rest_k,+ 32,+ 4,+ 4,+ dtype=torch.uint8,+ device=query.device,+ )+ if query_scales_second_scratch is None+ else query_scales_second_scratch[:rows]+ )+ native_scales_second = scale_storage_second.permute(+ 0, 2, 1, 3, 4, 5, 6+ ).permute(+ 4, 5, 2, 6, 3, 1, 0+ )+ query_scales_second = native_scales_second.as_strided(+ (+ 32,+ 4,+ 1,+ 4,+ rest_k,+ heads_kv,+ rows,+ ),+ native_scales_second.stride(),+ )++ quantize_decode_q_to_padded_fp4(+ query,+ query_fp4,+ query_scales,+ query_padded_out,+ query_fp4_out_2=query_fp4_second,+ query_scales_out_2=query_scales_second,+ row_indices=row_indices,+ heads_kv=heads_kv,+ scale_mode=scale_mode,+ rotation=rotation,+ tail_write=tail_write,+ )++ if query_terms == 2:+ assert query_fp4_second is not None+ assert query_scales_second is not None+ return (+ query_fp4,+ query_scales,+ query_fp4_second,+ query_scales_second,+ )+ return query_fp4, query_scalesdiff --git a/src/nvfp4_decode_kernel/fp4_decode_kernel.py b/src/nvfp4_decode_kernel/fp4_decode_kernel.pynew file mode 100644index 0000000..0b77335--- /dev/null+++ b/src/nvfp4_decode_kernel/fp4_decode_kernel.py@@ -0,0 +1,7953 @@+"""SM100 CuTeDSL kernel for hybrid FP4/BF16 paged decode."""++import enum+import math+from typing import Type, Tuple, Callable, Optional, Literal+from functools import partial+from dataclasses import dataclass+import cuda.bindings.driver as cuda++import cutlass+import cutlass.cute as cute+from cutlass import Float32, Int32, const_expr, Float8E4M3FN, Float4E2M1FN+from cutlass.cute.experimental import iket+from cutlass.cute.nvgpu import cpasync+import cutlass.cute.nvgpu.tcgen05 as tcgen05+from cutlass.cute.nvgpu.tcgen05.mma import MmaMXF4NVF4Op, MmaMXF4Op+import cutlass.utils.blackwell_helpers as sm100_utils_basic+from ._fa4.block_scaled_layout import make_smem_layout_sfa, make_smem_layout_sfb+import cutlass.utils.blockscaled_layout as blockscaled_utils+from ._fa4.paged_kv import PagedKVManager+from ._fa4 import utils as utils+from ._fa4 import copy_utils+from ._fa4 import pipeline as pipeline+from ._fa4.mask import AttentionMask+# from ._fa4.softmax import SoftmaxSm100, apply_score_mod_inner+from ._fa4.softmax import (+ SoftmaxSm100,+ _round_probability_code,+ _round_probability_scale,+ _amax2_or_amax6_probability_scale,+ _mxfp4_probability_exponent,+ apply_score_mod_inner,+)+from ._fa4.seqlen_info import SeqlenInfoQK+from ._fa4.block_info import BlockInfo+from ._fa4.block_sparsity import BlockSparseTensors+from ._fa4.block_sparse_utils import (+ get_total_block_count,+ produce_block_sparse_loads_sm100,+ softmax_block_sparse_sm100,+ handle_block_sparse_empty_tile_correction_sm100,+)+from ._fa4.pack_gqa import PackGQA+from ._fa4 import mma_sm100_desc as sm100_desc+from ._fa4 import blackwell_helpers as sm100_utils+from ._fa4.blackwell_helpers import packed_float_to_ue4m3, packed_float_to_e2m1+from ._fa4.fast_math import FastDivmod+from ._block_scale import inverse_power_of_two, round_up_exponent+from ._fa4.tile_scheduler import (+ TileSchedulerArguments,+ SingleTileScheduler,+ StaticPersistentTileScheduler,+ SingleTileLPTScheduler,+ SingleTileVarlenScheduler,+ ParamsBase,+)++def _sf_layout_with_page_stride(layout, page_stride):+ """Override the page pitch of a K/V block-scale layout.++ ``tile_to_shape`` derives every stride from the shape, which assumes the+ scale factors of consecutive pages sit back to back. When a page is a region+ of a wider cache page the pitch is larger, and it is the only stride that+ changes: the swizzle inside a page is fixed by the MMA. The page axis is+ mode 3, whose stride is ``(0, pitch)``.++ Reconstructing the layout drops the divisibility facts ``tile_to_shape``+ attached to the dynamic strides, so they are reapplied; the SFB TMA atom+ needs the 512-byte atom alignment to remain known.+ """+ if page_stride is None:+ return layout++ def assume_atom_aligned(value):+ if isinstance(value, int):+ return value+ return cute.assume(value, divby=512)++ stride = layout.stride+ return cute.make_layout(+ layout.shape,+ stride=(+ (stride[0][0], assume_atom_aligned(stride[0][1])),+ stride[1],+ (stride[2][0], assume_atom_aligned(stride[2][1])),+ (stride[3][0], assume_atom_aligned(page_stride)),+ ),+ )+++# A rescale whose factor is within 2**-RESCALE_THRESHOLD of one is skipped:+# the correction pass over O costs more than the exponent headroom it buys.+RESCALE_THRESHOLD = 8.0+++class NamedBarrierFwd(enum.IntEnum):+ Epilogue = enum.auto() # starts from 1 as barrier 0 is reserved for sync_threads()+ # Publishes the per-warp partial row maxima between the four softmax warps+ # of one warpgroup, once per KV block when S is transposed. The warpgroups+ # own disjoint query rows and must not synchronise with each other, so each+ # takes its own barrier; there is one slot per possible row group.+ SoftmaxReduce = enum.auto()+ SoftmaxReduce1 = enum.auto()+ SoftmaxReduce2 = enum.auto()+ SoftmaxReduce3 = enum.auto()+# WarpSchedulerWG1 = enum.auto()+# WarpSchedulerWG2 = enum.auto()+# WarpSchedulerWG3 = enum.auto()+# PFull = enum.auto()+# PEmpty = enum.auto()+++class SplitRole(enum.Enum):+ """Which blocks one CTA of the launch merges.++ ``ALL`` is every CTA when there is no split of its own for the BF16 blocks:+ the tail, the CTA's FP4 pages and the sink, each as far as it is fused.+ With one, the FP4 splits are ``PAGES``, their pages alone, and the last+ split is ``BF16``, the tail and the sink alone. Each role is a separately+ specialised body of every warp role, chosen once per CTA.+ """++ ALL = enum.auto()+ PAGES = enum.auto()+ BF16 = enum.auto()+++# How many ways the transposed softmax may split the query rows of a tile. The+# reduction takes one named barrier per group, counting up from SoftmaxReduce,+# so this many consecutive slots must exist in the enum above.+MAX_SOFTMAX_ROW_GROUPS = 4+assert (+ int(NamedBarrierFwd.SoftmaxReduce) + MAX_SOFTMAX_ROW_GROUPS - 1+ == int(NamedBarrierFwd.SoftmaxReduce3)+)++# A group runs one cross-warp reduction per KV tile — a butterfly, a named+# barrier and a round trip through shared memory — and that cost is per group,+# not per row, so it does not shrink as the group is handed fewer rows. Below+# this many rows a group spends more on its own reduction than the shorter+# serial chain saves: measured at batch 16, seqlen 128k, splitting four query+# rows four ways is 10 percent slower than splitting them two ways, while+# splitting eight and thirty-two rows four ways wins 5 and 10 percent.+MIN_ROWS_PER_SOFTMAX_GROUP = 2+++class FP4DecodeKernel:+ arch = 100++ def __init__(+ self,+ # dtype: Type[cutlass.Numeric],+ head_dim: int,+ head_dim_v: Optional[int] = None,+ qhead_per_kvhead: cutlass.Constexpr[int] = 1,+ is_causal: bool = False,+ is_local: bool = False,+ is_split_kv: bool = False,+ pack_gqa: bool = False,+ m_block_size: int = 128,+ n_block_size: int = 128,+ is_persistent: bool = True,+ score_mod: cutlass.Constexpr | None = None,+ mask_mod: cutlass.Constexpr | None = None,+ has_aux_tensors: cutlass.Constexpr = False,+ paged_kv_non_tma: bool = False,+ is_varlen_q: bool = False,+ sf_dtype: Optional[Type[cutlass.Numeric]] = None,+ sf_vec_size: Optional[int] = None,+ fused_residual_first_block: bool = False,+ fused_sink_block: bool = False,+ bf16_split: bool = False,+ residual_source: str = "contiguous",+ use_out_indices: bool = False,+ qhead_per_kvhead_O: cutlass.Constexpr[int] = None,+ seqlen_q_static_one: bool = False,+ transpose_s: bool = False,+ query_terms: cutlass.Constexpr[int] = 1,+ p_scale_divisor: cutlass.Constexpr[float] = 6.0,+ p_scale_grid_search: cutlass.Constexpr[bool] = False,+ p_scale_amax2_or_amax6: cutlass.Constexpr[bool] = False,+ p_scale_two_level: cutlass.Constexpr[bool] = False,+ p_scale_rescale_log2: cutlass.Constexpr[int] = 0,+ ):+ assert (sf_vec_size, sf_dtype) in (+ (16, cutlass.Float8E4M3FN), (32, cutlass.Float8E8M0FNU),+ ), "scales must be NVFP4 (16, E4M3) or MXFP4 (32, E8M0)"+ self.is_mxfp4 = sf_vec_size == 32+ assert not self.is_mxfp4 or (+ p_scale_divisor == 6.0 and not p_scale_grid_search+ and not p_scale_amax2_or_amax6 and not p_scale_two_level+ and p_scale_rescale_log2 == 0+ ), "MXFP4 P uses its own power-of-two scale search"+ assert query_terms in (1, 2)+ self.query_terms = query_terms+ assert p_scale_divisor in (2.0, 3.0, 4.0, 6.0)+ self.p_scale_divisor = p_scale_divisor+ self.p_scale_grid_search = p_scale_grid_search+ self.p_scale_amax2_or_amax6 = p_scale_amax2_or_amax6+ self.p_scale_two_level = p_scale_two_level+ if p_scale_rescale_log2 not in (0, 3):+ raise ValueError("P exponent offset must be zero or log2(8)")+ if p_scale_rescale_log2 and (not p_scale_amax2_or_amax6 or p_scale_two_level or p_scale_grid_search):+ raise ValueError("P exponent offset requires the amax2_or_amax6 ladder")+ self.p_scale_rescale_log2 = p_scale_rescale_log2+ assert not p_scale_two_level or (p_scale_divisor == 6.0 and not p_scale_grid_search and not p_scale_amax2_or_amax6)+ assert not (p_scale_grid_search and p_scale_amax2_or_amax6)+ self.fused_residual_first_block = fused_residual_first_block+ self.fused_sink_block = fused_sink_block+ assert not fused_sink_block or fused_residual_first_block, (+ "the BF16 sink reuses the residual cache and requires its prologue"+ )+ # The residual and the sink are single blocks outside the page range+ # split-k divides, so under split-k exactly one split may count them.+ # With bf16_split that is a split of their own, launched after the+ # FP4 splits: mO carries one more partial than there are page+ # partitions, the FP4 splits divide the pages among themselves alone,+ # and none of them waits for, loads or holds a BF16 block.+ # Without it every split runs both blocks and all but split 0 with a+ # length of zero, which the zero-length contract already makes+ # contribute nothing; split 0 is the one the scheduler always runs,+ # even when it holds no FP4 page.+ self.bf16_split = bf16_split+ assert not bf16_split or (is_split_kv and fused_residual_first_block), (+ "a split of its own for the BF16 blocks needs split-k and a residual"+ )+ self.residual_split_idx = 0+ assert residual_source in ("contiguous", "paged_bf16"), (+ f"unknown residual_source={residual_source!r}; "+ f"expected 'contiguous' or 'paged_bf16'"+ )+ self.residual_source = residual_source+ self.use_out_indices = use_out_indices+ self.seqlen_q_static_one = seqlen_q_static_one+ # Hold S as (kv position, query row) instead of (query row, kv position).+ # A tensor-memory load hands a thread one row and the same columns as+ # every other lane, so only the row axis can be narrowed per thread.+ # Decode leaves almost all of the query axis empty, and putting it in+ # the columns is what lets a thread exponentiate qhead_per_kvhead values+ # instead of a whole row of keys. The row reductions become cross-thread+ # in exchange. Resolved against the quantization flags in __call__.+ self.transpose_s_requested = transpose_s+ self.transpose_s = False+ self.tmem_sfpv_offset = None+ self.transposed_query_rows = 0+ self.stacked_query_terms = False+ self.narrow_pv = False+ self.pv_rows = 0+ self.softmax_row_groups = 1+ self.softmax_rows_per_group = 0+ self.softmax_scatters_rows = False+ self.softmax_red_slots = 0+ self.use_tma_KV = not paged_kv_non_tma+ # self.dtype = dtype+ # padding head_dim to a multiple of 16 as k_block_size+ hdim_multiple_of = 16+ self.head_dim_padded = int(math.ceil(head_dim / hdim_multiple_of) * hdim_multiple_of)+ head_dim_v = head_dim_v if head_dim_v is not None else head_dim+ self.same_hdim_kv = head_dim == head_dim_v+ self.head_dim_v_padded = int(math.ceil(head_dim_v / hdim_multiple_of) * hdim_multiple_of)+ self.same_hdim_kv_padded = self.head_dim_padded == self.head_dim_v_padded+ self.check_hdim_oob = head_dim != self.head_dim_padded+ self.check_hdim_v_oob = head_dim_v != self.head_dim_v_padded+ self.m_block_size = m_block_size+ self.n_block_size = n_block_size+ # A second Q stage only earns its keep when a CTA carries two Q tiles.+ # With seqlen_q statically one the packed M extent is qhead_per_kvhead+ # rows under PackGQA and a single row without it, either way at most one+ # m_block_size, so stage 1 would TMA an out-of-range Q, get zeros, and+ # run a full softmax, TMEM round trip and PV MMA over them.+ if seqlen_q_static_one:+ assert qhead_per_kvhead <= m_block_size+ if is_split_kv or seqlen_q_static_one:+ self.q_stage = 1+ elif getattr(type(self), "_force_q_stage_1", False):+ self.q_stage = 1+ else:+ self.q_stage = 2+ assert self.q_stage in [1, 2]+ assert self.query_terms == 1 or self.q_stage == 1+ # 2 Q tile per CTA+ self.cta_tiler = (self.q_stage * m_block_size, n_block_size, self.head_dim_padded)+ self.mma_tiler_qk = (m_block_size, n_block_size, self.head_dim_padded)+ self.mma_tiler_pv = (m_block_size, self.head_dim_v_padded, n_block_size)+ self.qk_acc_dtype = Float32+ self.pv_acc_dtype = Float32+ self.cluster_shape_mn = (1, 1)+ self.is_persistent = is_persistent+ self.is_causal = is_causal+ self.is_local = is_local+ self.is_varlen_q = is_varlen_q+ self.use_correction_warps_for_epi = is_varlen_q+ self.qhead_per_kvhead = qhead_per_kvhead+ self.qhead_per_kvhead_O = (+ qhead_per_kvhead_O if qhead_per_kvhead_O is not None else qhead_per_kvhead+ )+ assert self.qhead_per_kvhead_O >= self.qhead_per_kvhead, (+ "qhead_per_kvhead_O must be >= qhead_per_kvhead (pad up, never down)"+ )+ self.is_split_kv = is_split_kv+ self.pack_gqa = pack_gqa+ # A single query position packs onto a flat head axis (see the PackGQA+ # layouts in __call__), which tiles for any ratio up to m_block_size.+ # Only the (head, seqlen) packing needs the tile to be a whole number+ # of query groups.+ if pack_gqa and not seqlen_q_static_one:+ assert m_block_size % self.qhead_per_kvhead == 0, (+ "For PackGQA, m_block_size must be divisible by qhead_per_kvhead"+ )+ assert m_block_size % self.qhead_per_kvhead_O == 0, (+ "For PackGQA with separate O-axis, m_block_size must be divisible by qhead_per_kvhead_O"+ )+ # The MMA fixes the M extent at 128 rows while decode fills only+ # qhead_per_kvhead of them, and a softmax thread owns one row, so the+ # rows that carry no query still cost a full row of exp2. Repeating each+ # query row across the tile puts a real row under every thread, which is+ # what lets the column range a thread owns shrink by this factor. The+ # copies of one query row are consecutive, so a group of them lands+ # inside a single warp and the row reductions stay warp-local.+ # The split path writes FP32 partials and an LSE per row for the combine+ # kernel to consume, both keyed by the unreplicated row order, so it is+ # left alone until those two writers learn the same row selection the+ # unsplit epilogue uses.+ self.q_replicate = 1+ if (+ pack_gqa+ and seqlen_q_static_one+ and not is_split_kv+ and getattr(type(self), "_enable_q_replicate", False)+ ):+ self.q_replicate = m_block_size // self.qhead_per_kvhead+ assert not (self.is_split_kv and self.head_dim_v_padded >= 192), (+ "SplitKV is not supported for hdim >= 192"+ )+ self.score_mod = score_mod+ self.mask_mod = mask_mod+ if cutlass.const_expr(has_aux_tensors):+ self.vec_size: cutlass.Constexpr = 1+ else:+ self.vec_size: cutlass.Constexpr = 2+ # Does S1 need to wait for S0 to finish+ # self.s0_s1_barrier = self.head_dim_padded in [64, 96] and (not self.is_causal and not self.is_local)+ self.s0_s1_barrier = False+ self.overlap_sO_sQ = (+ (self.head_dim_padded == 192 and self.head_dim_v_padded >= 64) or+ (self.head_dim_v_padded >= 128 and self.is_split_kv)+ )+ # BF16 blocks merged ahead of and behind a CTA's own FP4 pages hold+ # their tiles until the tile's last MMA, and O is staged only once that+ # MMA has completed, so on a CTA that owns one tile O takes the tiles'+ # bytes instead of widening sQ or holding its own.+ self.overlap_sO_bf16 = fused_residual_first_block and not bf16_split+ if self.overlap_sO_bf16:+ assert not is_persistent, (+ "O staged in the BF16 tiles needs one tile per CTA"+ )+ self.overlap_sO_sQ = False+ if self.overlap_sO_sQ:+ self.is_persistent = False++ assert self.use_tma_KV or not (self.check_hdim_oob or self.check_hdim_v_oob), (+ "Paged KV does not support irregular head dim"+ )++ SM100_TMEM_CAPACITY_COLUMNS = 512+ self.tmem_alloc_cols = SM100_TMEM_CAPACITY_COLUMNS++ self.tmem_s_offset = [0, self.n_block_size] # e.g., 0, 128+ self.tmem_o_offset = [+ self.tmem_s_offset[-1] + self.n_block_size + i * self.head_dim_v_padded+ for i in range(self.q_stage)+ ] # e.g., 256, 384+ # self.tmem_o_offset = self.tmem_s_offset+ self.tmem_total = self.tmem_o_offset[-1] + self.head_dim_v_padded+ assert self.tmem_total <= SM100_TMEM_CAPACITY_COLUMNS+ self.tmem_s_to_p_offset = self.n_block_size // 2++ self.tmem_p_offset = [+ self.tmem_s_offset[i] + self.tmem_s_to_p_offset for i in range(2)+ ] # e.g., 64, 192++ self.tmem_p_bf16_offset = [+ self.tmem_s_offset[i] + self.tmem_s_to_p_offset for i in range(2)+ ] # same as FP4 but aliased temporally — see comment above.++ # vec buffer for row_max & row_sum+ self.tmem_vec_offset = self.tmem_s_offset++ # The count a softmax warpgroup was tuned at. A CTA wide enough that it+ # cannot afford this narrows it; see _assign_warp_roles.+ if self.head_dim_padded < 96:+ self.num_regs_softmax_tuned = 200+ self.num_regs_correction = 64+ self.num_regs_other = 48+ else:+ # self.num_regs_softmax_tuned = 192 if self.is_causal or self.is_local else 184+ self.num_regs_softmax_tuned = 216+ # self.num_regs_softmax_tuned = 176+ # self.num_regs_correction = 96+ # self.num_regs_correction = 80+ # self.num_regs_correction = 64 if self.is_causal or self.is_local else 80+ self.num_regs_correction = 48+ # self.num_regs_other = 32+ # self.num_regs_other = 64+ # self.num_regs_other = 80+ self.num_regs_other = 24+ # self.num_regs_other = 96 if self.is_causal or self.is_local else 80+ # self.num_regs_other = 64 if self.is_causal or self.is_local else 80+ self.num_regs_empty = 24+ # Two softmax warpgroups is the layout every configuration starts from:+ # one per Q stage. A transposed decode that splits its query rows four+ # ways asks for four and re-runs this once __call__ has resolved that.+ self._assign_warp_roles(2)+ self.buffer_align_bytes = 1024++ # Scale factor parameters for block-scaled quantization (FP4)+ self.sf_dtype = sf_dtype+ self.sf_vec_size = sf_vec_size+ self.mma_inst_bits_k = 256+ self.mma_inst_tile_k = self.head_dim_padded // (self.mma_inst_bits_k // 4)+ self.sf_groups_per_mma_k = self.mma_inst_bits_k // 4 // self.sf_vec_size++ # An SM holds this many 32-bit registers, handed out in blocks of eight per+ # thread, and the kernel runs one CTA per SM.+ REGISTER_FILE_PER_SM = 65536+ REGISTER_ALLOC_GRANULARITY = 8+ WARPS_PER_WARPGROUP = 4++ def _assign_warp_roles(self, num_softmax_warpgroups: int):+ """Lay out the warp roles for a CTA with this many softmax warpgroups.++ Each softmax warpgroup is a contiguous run of four warps starting on a+ 128-thread boundary, which is what makes ``thread_idx % 128`` the kv+ position inside the tile that the transposed softmax indexes by. The+ remaining roles follow: one correction warpgroup, then the mma,+ epilogue, load and idle warps that share the last warpgroup.++ Widening the CTA costs registers, so the softmax warpgroups split+ whatever the correction and service warpgroups leave of the CTA's+ register allocation rather than each keeping the count a narrower CTA+ could afford.+ """+ warps = self.WARPS_PER_WARPGROUP+ threads_per_warpgroup = warps * cute.arch.WARP_SIZE+ self.softmax_warp_ids = tuple(+ tuple(range(g * warps, (g + 1) * warps))+ for g in range(num_softmax_warpgroups)+ )+ self.softmax0_warp_ids = self.softmax_warp_ids[0] # stage 0+ self.softmax1_warp_ids = self.softmax_warp_ids[1] # stage 1+ service = num_softmax_warpgroups * warps+ self.correction_warp_ids = tuple(range(service, service + warps))+ self.mma_warp_id = service + warps+ self.epilogue_warp_ids = (service + warps + 1,)+ self.load_warp_ids = (service + warps + 2,)+ self.empty_warp_ids = (service + warps + 3,)+ self.threads_per_cta = cute.arch.WARP_SIZE * (service + 2 * warps)++ if not self.use_tma_KV:+ self.load_warp_ids = self.load_warp_ids + self.empty_warp_ids+ self.empty_warp_ids = ()+ if self.use_correction_warps_for_epi:+ self.empty_warp_ids = self.empty_warp_ids + self.epilogue_warp_ids+ self.epilogue_warp_ids = self.correction_warp_ids+ elif self.is_varlen_q: # fallback+ self.epilogue_warp_ids = (service + warps + 1, service + warps + 2)++ # setmaxnreg only redistributes the registers the CTA was given at+ # launch, which is threads_per_cta times the largest per-thread count+ # that fits the register file, rounded down to the allocation+ # granularity. A CTA of 768 threads loses that rounding twice over: the+ # per-thread count drops to 80 and the CTA's pool with it, so the+ # arithmetic has to be done against the pool and not against the whole+ # register file. Over-subscribe it and setmaxnreg.inc waits for+ # registers no one will release.+ regs_at_launch = (+ self.REGISTER_FILE_PER_SM+ // self.threads_per_cta+ // self.REGISTER_ALLOC_GRANULARITY+ * self.REGISTER_ALLOC_GRANULARITY+ )+ softmax_pool = regs_at_launch * self.threads_per_cta - threads_per_warpgroup * (+ self.num_regs_correction + max(self.num_regs_other, self.num_regs_empty)+ )+ affordable = (+ softmax_pool+ // (threads_per_warpgroup * num_softmax_warpgroups)+ // self.REGISTER_ALLOC_GRANULARITY+ * self.REGISTER_ALLOC_GRANULARITY+ )+ self.num_regs_softmax = min(self.num_regs_softmax_tuned, affordable)++ def _softmax_row_group_count(self) -> int:+ """How many ways the transposed softmax splits this tile's query rows.++ A decode carries one Q tile, so only the first softmax warpgroup has a+ stage to run. Spreading the query rows over more groups shortens the+ serial chain every thread runs — its butterfly, its exponentials and+ its packing — and the groups write disjoint rows of P and of the+ scales. Two groups is what any tile with more than one query row+ already split into; going past two is worth it only while every group+ keeps enough rows to amortize its own reduction.++ Plain Python, evaluated while ``__call__`` is traced, so the count and+ everything derived from it are compile-time constants.+ """+ # The BF16 steps share one P tile between the groups, and nothing+ # orders one group's stores against another's clear, so each clears+ # only rows no group writes (see zero_transposed_p_bf16). A second Q+ # stage gives the second warpgroup its own work.+ if self.q_stage != 1:+ return 1+ if self.transposed_query_rows < 2:+ return 1+ return min(+ MAX_SOFTMAX_ROW_GROUPS,+ max(2, self.transposed_query_rows // MIN_ROWS_PER_SOFTMAX_GROUP),+ )++ def _setup_attributes(self):+ """Set up configurations and parameters for the FMHA kernel operation.++ This method initializes and configures various attributes required for the+ execution of the fused multi-head attention kernel, mainly about the pipeline stages:++ - Sets up staging parameters for Q, K, V inputs and accumulator data+ - Configures pipeline stages for softmax, correction, and epilogue operations+ """+ self.acc_stage = 1+ # The epilogue indexes sO by q tile, so a deeper ring than there are q+ # tiles is memory no one can name. Decode runs a single q tile, where+ # the second buffer was costing 32KB of output, or 64KB once the split+ # path widens it to fp32 partials.+ self.epi_stage = self.q_stage+ # Compute kv_stage from SMEM budget.+ # Blackwell: 228KB per SM, 227KB optin per block.+ # K and V alias when same dtype or when K is smaller (FP4 K in BF16 V).+ smem_budget = 227 * 1024+ align = self.buffer_align_bytes # 128B struct field alignment+ def align_up(x, a): return (x + a - 1) // a * a+ # Fixed fields (not scaled by kv_stage): mbar, tmem_holding_buf, sScale, sO, sQ, SFQ, SFP+ # mbar_total depends on kv_stage but is small (~40 barriers * 8B = 320B); use upper bound+ smem_mbar = 512 # generous upper bound for mbarrier storage+ smem_tmem = 4 # Int32+ smem_sScale = align_up(max(self.q_stage * self.m_block_size * 2, self.m_block_size * 4) * 4, align) # Float32+ smem_q_per_stage = self.m_block_size * self.head_dim_padded * self.q_dtype.width // 8+ smem_o_per_stage = self.m_block_size * self.head_dim_v_padded * self.o_dtype.width // 8+ # SFQ/SFP are per q_stage, SF layout cosize: m_block * head_dim / sf_vec_size+ sfq_per_stage = self.m_block_size * self.head_dim_padded // self.sf_vec_size+ sfp_per_stage = self.m_block_size * self.head_dim_v_padded // self.sf_vec_size+ sfq_bytes = (+ sfq_per_stage * self.q_stage * self.query_terms if self.quant_qk else 0+ )+ # When the two overlap, sQ is widened to cover sO rather than sO being+ # dropped, so the budget has to charge the wider of the two. The Q side+ # is every query term's tile and its scale factors, which are dead by+ # the time sO is written, just like the tiles.+ if self.overlap_sO_sQ:+ smem_sO = 0+ smem_sQ = align_up(+ max(+ smem_q_per_stage * self.q_stage * self.query_terms + sfq_bytes,+ smem_o_per_stage * self.epi_stage,+ ),+ align,+ )+ smem_sSFQ = 0+ else:+ smem_sO = align_up(smem_o_per_stage * self.epi_stage, align)+ smem_sQ = align_up(+ smem_q_per_stage * self.q_stage * self.query_terms,+ align,+ )+ smem_sSFQ = align_up(sfq_bytes, align)+ smem_sSFP = align_up(sfp_per_stage * self.q_stage, align) if self.quant_pv else 0+ # A transposed P reaches the tensor core from shared memory, and the+ # cross-warp row maximum needs one float per softmax warp per query row,+ # double buffered so a single named barrier per KV block suffices.+ if self.transpose_s:+ smem_sP = align_up(+ self.m_block_size * self.n_block_size * self.v_dtype.width // 8, align+ )+ smem_sRed = align_up(self.softmax_red_slots * 4, align)+ else:+ smem_sP = 0+ smem_sRed = 0+ if self.fused_residual_first_block:+ smem_sK_bf16 = align_up(self.m_block_size * self.head_dim_padded * 2, align)+ smem_sV_bf16 = align_up(self.m_block_size * self.head_dim_v_padded * 2, align)+ smem_sQ_bf16 = align_up(self.m_block_size * self.head_dim_padded * 2, align)+ else:+ smem_sK_bf16 = 0+ smem_sV_bf16 = 0+ smem_sQ_bf16 = 0+ smem_bf16_tiles = smem_sK_bf16 + smem_sV_bf16 + smem_sQ_bf16+ if self.overlap_sO_bf16:+ smem_bf16_tiles = max(smem_bf16_tiles, smem_sO)+ smem_sO = 0+ # On a split of their own the BF16 tiles share their bytes with the+ # K/V stages (see SharedStorage), so they are charged further down.+ smem_fixed = smem_mbar + smem_tmem + smem_sScale + smem_sO + smem_sQ + smem_sSFQ + smem_sSFP + smem_sP + smem_sRed+ if not self.bf16_split:+ smem_fixed += smem_bf16_tiles+ # Per-kv_stage fields: sK (or aliased), sV, SFK, SFV+ smem_k_per_stage = self.m_block_size * self.head_dim_padded * self.k_dtype.width // 8+ smem_v_per_stage = self.m_block_size * self.head_dim_v_padded * self.v_dtype.width // 8+ if self.v_dtype == self.k_dtype or self.k_dtype.width < self.v_dtype.width:+ smem_kv_per_stage = max(smem_k_per_stage, smem_v_per_stage)+ else:+ smem_kv_per_stage = smem_k_per_stage + smem_v_per_stage+ # SF layout cosize per stage: n_block * head_dim / sf_vec_size (MMA-tiled)+ sfk_per_stage = self.n_block_size * self.head_dim_padded // self.sf_vec_size+ sfv_per_stage = self.n_block_size * self.head_dim_v_padded // self.sf_vec_size+ if self.quant_qk:+ smem_kv_per_stage += sfk_per_stage+ if self.quant_pv:+ smem_kv_per_stage += sfv_per_stage+ # Add per-stage padding for swizzle/layout cosize inflation (~128B per staged field)+ num_kv_staged_fields = 2 # sK + sV (or 1 if aliased, but sV still has cosize overhead)+ if self.quant_qk:+ num_kv_staged_fields += 1 # sSFK+ if self.quant_pv:+ num_kv_staged_fields += 1 # sSFV+ smem_kv_per_stage += num_kv_staged_fields * 128+ self.kv_stage = (smem_budget - smem_fixed) // smem_kv_per_stage+ if self.quant_qk and self.quant_pv:+ # Depth buys latency hiding only until the mainloop stops waiting on+ # KV. Past that the extra buffers cost cosize and issue slots: on the+ # non-split path a depth sweep is flat from 8 to 10 and 1% worse at+ # 12 and beyond, so spending the whole SMEM budget (which lands at+ # 14) is a loss. The split path carries an FP32 partial-output buffer+ # and cannot fit more than 9 stages anyway.+ KV_STAGE_FP4_SPLITK_CAP = 8+ KV_STAGE_FP4_CAP = 10+ cap = KV_STAGE_FP4_SPLITK_CAP if self.is_split_kv else KV_STAGE_FP4_CAP+ self.kv_stage = min(self.kv_stage, cap)+ if self.bf16_split:+ # The tiles overlay the stages' K/V data, so only what they need+ # past it adds to the budget.+ smem_kv_data_per_stage = max(smem_k_per_stage, smem_v_per_stage)++ def smem_total(stages):+ overhang = smem_bf16_tiles - stages * smem_kv_data_per_stage+ return smem_fixed + stages * smem_kv_per_stage + max(0, overhang)++ while smem_total(self.kv_stage) > smem_budget:+ self.kv_stage -= 1+ assert self.kv_stage >= 2, (+ f"kv_stage={self.kv_stage} < 2: FP4 mainloop requires K/V double-buffer"+ )+ # For hdim 192,128, we don't have enough smem to store all 3 stages of KV:+ # 128 x 192 x 2 bytes x 3 stages = 144KB, and we need 96KB for Q.+ # Instead we store smem as [smem_large, smem_small, smem_large], where smem_large is+ # 128 x 192 and smem_small is 128 x 128. We set the stride between the stages to be+ # 128 * 160, so that indexing the 0th and 2nd stages will get the right address,+ # but for the 1st stage we need to add or subtract (depending on phase) 128 x 64.+ # self.uneven_kv_smem = (+ # self.head_dim_padded == 192 and self.head_dim_v_padded == 128 and self.kv_stage == 3+ # )+ self.uneven_kv_smem = False+ self.uneven_kv_smem_offset = (+ self.m_block_size * (self.head_dim_padded - self.head_dim_v_padded) // 2+ if self.uneven_kv_smem+ else 0+ )+ assert self.uneven_kv_smem_offset % 1024 == 0++ @cute.jit+ def __call__(+ self,+ mQ, # cute.Tensor or cute.Pointer (b, s_q, h, d)+ mK, # cute.Tensor or cute.Pointer (b_k, s_k, h_k, d)+ mV, # cute.Tensor or cute.Pointer (b_k, s_k, h_k, dv)+ mO: cute.Tensor, # (b, s_q, h, dv)+ mLSE: Optional[cute.Tensor],+ softmax_scale: Float32,+ stream: cuda.CUstream,+ mCuSeqlensQ: Optional[cute.Tensor] = None,+ mCuSeqlensK: Optional[cute.Tensor] = None,+ mSeqUsedQ: Optional[cute.Tensor] = None,+ mSeqUsedK: Optional[cute.Tensor] = None,+ mPageTable: Optional[cute.Tensor] = None,+ window_size_left: Int32 | int | None = None,+ window_size_right: Int32 | int | None = None,+ learnable_sink: Optional[cute.Tensor] = None,+ blocksparse_tensors: Optional[BlockSparseTensors] = None,+ aux_tensors: Optional[list] = None,+ mSFQ: Optional[cute.Tensor] = None,+ mSFK: Optional[cute.Tensor] = None,+ mSFV: Optional[cute.Tensor] = None,+ # For pointer-based Q/K: separate shapes to handle cross-attention (seqlen_q != seqlen_k)+ q_ptr_shape: tuple = (),+ k_ptr_shape: tuple = (),+ # For pointer-based V (FP4 K-major): full headdim shape (b, s, h, d)+ v_ptr_shape: tuple = (),+ # Distance between consecutive K/V pages, in FP4 elements for the packed+ # data and in bytes for the scale factors. None means the pages are+ # densely packed, which is what the shapes alone imply. A larger stride+ # lets one page be a region of a wider cache page that also holds the+ # scale factors and the other of K/V.+ k_page_stride=None,+ v_page_stride=None,+ k_sf_page_stride=None,+ v_sf_page_stride=None,+ compute_sp1: cutlass.Constexpr[bool] = False,+ mResidualQ: Optional[cute.Tensor] = None,+ mResidualK: Optional[cute.Tensor] = None,+ mResidualV: Optional[cute.Tensor] = None,+ mResidualSeqUsedK: Optional[cute.Tensor] = None,+ mResidualKCache: Optional[cute.Tensor] = None,+ mResidualVCache: Optional[cute.Tensor] = None,+ mResidualBlockIds: Optional[cute.Tensor] = None,+ mSinkSeqUsedK: Optional[cute.Tensor] = None,+ mOutIndices: Optional[cute.Tensor] = None,+ mQSecond=None,+ mSFQSecond: Optional[cute.Tensor] = None,+ ):+ """Execute the Fused Multi-Head Attention operation on the provided tensors.++ For FP4, mQ/mK can be cute.Pointer with q/k_ptr_shape providing (b, s, h, d).+ The kernel builds tensors from the pointer using make_ordered_layout.+ """+ # Build Q/K tensors from pointer/tensor + shape+ # For pointers: mQ is a Pointer, .iterator not needed+ # For tensors: mQ is a Tensor, use .iterator to extract pointer+ q_iter = mQ.iterator if hasattr(mQ, 'iterator') else mQ+ q_second_iter = (+ mQSecond.iterator if hasattr(mQSecond, "iterator") else mQSecond+ )+ k_iter = mK.iterator if hasattr(mK, 'iterator') else mK+ mQ = cute.make_tensor(q_iter, cute.make_ordered_layout(+ q_ptr_shape, order=tuple(range(len(q_ptr_shape) - 1, -1, -1))+ ))+ mQSecond = cute.make_tensor(+ q_second_iter,+ cute.make_ordered_layout(+ q_ptr_shape,+ order=tuple(range(len(q_ptr_shape) - 1, -1, -1)),+ ),+ )+ if const_expr(k_page_stride is None):+ mK = cute.make_tensor(k_iter, cute.make_ordered_layout(+ k_ptr_shape, order=tuple(range(len(k_ptr_shape) - 1, -1, -1))+ ))+ else:+ # Same row-major (b, s, h, d) layout the ordered form produces, with+ # the page stride supplied instead of derived. The innermost stride+ # stays a literal 1 so the major mode is still statically known.+ _, _, k_h, k_d = k_ptr_shape+ mK = cute.make_tensor(k_iter, cute.make_layout(+ k_ptr_shape, stride=(k_page_stride, k_h * k_d, k_d, 1)+ ))+ # FP4 K-major V: build from pointer with explicit (b, s, h, d) shape and+ # K-major strides (S*H*D, 1, S, S*H). The host transposes V's underlying+ # buffer so that seqlen has stride 1 in the FP4 byte buffer.+ if const_expr(len(v_ptr_shape) > 0):+ v_iter = mV.iterator if hasattr(mV, 'iterator') else mV+ # K-major V: nvfp4_quantize on `v.permute(0,2,3,1).reshape(b*h*d, s)`+ # produces an FP4 byte buffer of physical shape (b, h, d, s/2) row-major,+ # where each int8 byte holds two seqlen-adjacent FP4 in the high/low+ # nibble. Logically the V tensor has FP4 shape (b, s, h, d) with element+ # strides (h*d*s, 1, h*d, d) — order=(3, 0, 2, 1):+ # s (dim 1) order 0 → stride 1+ # d (dim 3) order 1 → stride s+ # h (dim 2) order 2 → stride s*d+ # b (dim 0) order 3 → stride s*d*h+ # Int64 shape so make_ordered_layout produces Int64 strides+ # (tma_partition for SFV requires Int64).+ from cutlass import Int64+ v_b, v_s, v_h, v_d = v_ptr_shape+ v_shape = (Int64(v_b), Int64(v_s), Int64(v_h), Int64(v_d))+ if const_expr(v_page_stride is None):+ mV = cute.make_tensor(v_iter, cute.make_ordered_layout(+ v_shape, order=(3, 0, 2, 1),+ ))+ else:+ # The order=(3, 0, 2, 1) strides written out, with the page+ # stride supplied. Seqlen keeps a literal stride of 1 so V stays+ # statically K-major.+ mV = cute.make_tensor(v_iter, cute.make_layout(+ v_shape,+ stride=(+ Int64(v_page_stride),+ 1,+ Int64(v_s) * Int64(v_d),+ Int64(v_s),+ ),+ ))+ self.q_dtype = mQ.element_type+ self.k_dtype = mK.element_type+ self.v_dtype = mV.element_type+ self.o_dtype = mO.element_type+ self.compute_sp1 = const_expr(compute_sp1)+ if const_expr(mSFQ is None):+ assert self.q_dtype.width >= 8+ assert const_expr(mSFK is None), "Must provide both QK sfs or None"+ self.quant_qk = const_expr(mSFQ is not None)+ self.quant_pv = const_expr(mSFV is not None)+ assert not (not self.quant_qk and self.quant_pv)++ # The transposed layout only pays off where the query axis is short and+ # statically known, and it reroutes P through shared memory, so it is+ # confined to the pure FP4 decode shape. Everything else keeps the+ # untransposed path. The query count is padded to a power of two below,+ # because it becomes the repetition count of a single tcgen05 load.+ # The conditions are combined with all() over a list rather than a chain+ # of `and`. Preprocessing rewrites each short-circuit operator into a+ # nested conditional that carries a copy of the continuation, so an+ # n-term chain in a traced function grows the transformed AST by 2**n;+ # sixteen terms here took this function from 14k nodes to over 4M and+ # its preprocessing from 0.01s to more than ten minutes. Every term is a+ # plain attribute read, so evaluating all of them eagerly is free.+ self.transpose_s = const_expr(+ all(+ [+ self.transpose_s_requested,+ self.quant_qk,+ self.quant_pv,+ self.pack_gqa,+ self.seqlen_q_static_one,+ not self.compute_sp1,+ not self.is_causal,+ not self.is_local,+ # self.use_block_sparsity is not derived until much later in+ # this function, so read its source argument directly.+ blocksparse_tensors is None,+ self.score_mod is None,+ self.qhead_per_kvhead <= 128,+ ]+ )+ )+ assert not self.p_scale_two_level or self.transpose_s, "two_level P requires transposed decode"+ assert not self.p_scale_rescale_log2 or self.transpose_s, (+ "rescaled P requires transposed decode"+ )+ assert self.query_terms == 1 or self.transpose_s, (+ "two-term query decode requires the transposed score path"+ )+ assert not self.is_mxfp4 or self.transpose_s, (+ "MXFP4 decode requires the transposed score path"+ )+ if const_expr(self.transpose_s):+ # Replication addresses the same padding by repeating rows, which+ # the transposed layout makes pointless.+ self.q_replicate = 1+ # Transposing swaps the two leading modes of the QK tiler. Requiring+ # them equal keeps that tuple, and every partition derived from it,+ # unchanged, so only the operand roles move.+ assert self.m_block_size == self.n_block_size, (+ "transpose_s requires a square QK tile"+ )+ # PackGQA with a statically-one query length fills qhead_per_kvhead+ # rows of the M tile; transposed, they are the live columns of S.+ # The count becomes the repetition of a single tcgen05 load, which+ # must be a power of two, so it is rounded up. The extra columns+ # carry the zero rows TMA fills past the flat head axis: their+ # scores are finite and their rows of O are never stored.+ self.transposed_query_rows = 1 << (self.qhead_per_kvhead - 1).bit_length()+ # Two independently scaled terms occupy disjoint query rows of+ # one narrow QK instruction. The logical softmax row count stays+ # unchanged; its FP32 addition joins the two score columns.+ self.stacked_query_terms = (+ self.query_terms == 2+ and self.q_stage == 1+ and self.qhead_per_kvhead in (4, 8, 16)+ )+ # PV accumulates O^T: V is the A operand over the head dimension and+ # P the B operand over the live query rows, rounded up to the eight+ # an MMA needs. Padding the rows to the 128 of an A operand instead+ # spends sixteen times the tensor-core energy at a ratio of eight,+ # which under the power cap costs clock. Each correction thread then+ # holds one head-dimension lane of every live row, so the row count+ # is bounded by the correction warps' registers; the two-level P+ # scale accumulates rows of O per thread and keeps the wide tile.+ self.pv_rows = max(8, self.transposed_query_rows)+ self.narrow_pv = all(+ [+ self.transposed_query_rows <= 16,+ self.q_stage == 1,+ not self.p_scale_two_level,+ learnable_sink is None,+ self.head_dim_v_padded == self.m_block_size,+ not self.use_correction_warps_for_epi,+ ]+ )+ self.softmax_row_groups = self._softmax_row_group_count()+ # transposed_query_rows is a power of two, so the group count+ # always divides it evenly.+ self.softmax_rows_per_group = (+ self.transposed_query_rows // self.softmax_row_groups+ )+ # The row maximum reduce-scatters a thread's rows across lanes when+ # it holds more than one and each lane's row still has the lanes of+ # a whole scale group to itself. The grid-search P scale reduces an+ # error per row over the group and keeps the row-per-thread path.+ self.softmax_scatters_rows = (+ 2 <= self.softmax_rows_per_group <= self.sf_vec_size+ and not self.p_scale_grid_search+ )+ # One softmax warpgroup per row group, except that the CTA never+ # narrows below the two warpgroups the Q stages need.+ self._assign_warp_roles(max(2, self.softmax_row_groups))+ # Two buffers per warp per row, so a single named barrier per KV+ # block suffices for the row reductions. Splitting the rows across+ # groups moves slots between the groups without changing the total.+ self.softmax_red_slots = (+ 2 * len(self.softmax0_warp_ids) * self.transposed_query_rows+ )+ # P and its scale factors normally alias the tensor memory S was+ # read out of, which is why the QK of the next block has to wait+ # behind this block's PV. Transposed, P lives in shared memory, so+ # only the scale factors still alias, and a decode leaves 128+ # tensor-memory columns free above O to move them into. Once they+ # are out of the way the only thing the next QK waits on is the+ # softmax having drained S, which it signals as soon as the load+ # completes rather than at the end of the step.+ self.tmem_sfpv_offset = self.tmem_total+ if const_expr(self.fused_sink_block):+ assert self.transpose_s, (+ "the two-island BF16 path requires transposed scores"+ )+ # Each CTA picks its role once, from its split, so it must own exactly+ # one tile for its whole life. Its BF16 tiles share the bytes of sK,+ # which therefore has to hold V as well.+ if const_expr(self.bf16_split):+ assert (+ self.transpose_s+ and not self.is_persistent+ and self.v_dtype == self.k_dtype+ ), (+ "a split of its own for the BF16 blocks needs transposed "+ "scores, one tile per CTA, and K and V of one dtype"+ )++ # Assume all strides are divisible by 128 bits except the last stride+ def _assume_strides(t):+ divby = 128 // t.element_type.width+ return tuple(+ s if isinstance(s, int) else cute.assume(s, divby=divby)+ for s in t.stride[:-1]+ ) + (t.stride[-1],)+ mV, mO = [+ cute.make_tensor(t.iterator, cute.make_layout(t.shape, stride=_assume_strides(t)))+ for t in (mV, mO)+ ]+ Q_layout_transpose = [1, 3, 2, 0] if const_expr(mCuSeqlensQ is None) else [0, 2, 1]+ mQ = cute.make_tensor(mQ.iterator, cute.select(mQ.layout, mode=Q_layout_transpose)) # (s_q, d, h, b)+ mQSecond = cute.make_tensor(+ mQSecond.iterator,+ cute.select(mQSecond.layout, mode=Q_layout_transpose),+ )+ # (s_k, d, h_k, b_k) or (total_k, d, h_k) if there's cu_seqlens_k or (page_size, d, h_k, num_pages) if there's page_table+ KV_layout_transpose = [1, 3, 2, 0] if const_expr(mCuSeqlensK is None) else [0, 2, 1]+ mK, mV = [+ cute.make_tensor(t.iterator, cute.select(t.layout, mode=KV_layout_transpose))+ for t in (mK, mV)+ ]+ if const_expr(self.is_split_kv):+ O_layout_transpose = [2, 4, 3, 1, 0] if const_expr(mCuSeqlensQ is None) else [1, 3, 2, 0]+ LSE_layout_transpose = [3, 2, 1, 0] if const_expr(mCuSeqlensQ is None) else [2, 1, 0]+ num_splits = mO.shape[0]+ else:+ O_layout_transpose = [1, 3, 2, 0] if const_expr(mCuSeqlensQ is None) else [0, 2, 1]+ LSE_layout_transpose = [2, 1, 0] if const_expr(mCuSeqlensQ is None) else [1, 0]+ num_splits = Int32(1)+ mO = cute.make_tensor(mO.iterator, cute.select(mO.layout, mode=O_layout_transpose))+ mLSE = (+ cute.make_tensor(mLSE.iterator, cute.select(mLSE.layout, mode=LSE_layout_transpose))+ if const_expr(mLSE is not None)+ else None+ )+ # (s, d, h, b) -> (d, s, h, b)+ # For FP4 block-scaled MMA, B (V) must be K-major (K=seqlen contiguous).+ # Skip V transpose when V is already K-major (headdim contiguous = N contiguous after transpose).+ # Without transpose: mV = (s, d, h, b) → mode 0=s(K), mode 1=d(N) with d contiguous → K-major: NO!+ # With transpose: mV = (d, s, h, b) → mode 0=d(N), mode 1=s(K) with d contiguous → MN-major+ # For FP4 we need K-major: mode 1 (K=seqlen) contiguous. Need V physically transposed on host.+ V_layout_transpose = [1, 0, 2, 3] if const_expr(mCuSeqlensK is None) else [1, 0, 2]+ mV = cute.make_tensor(mV.iterator, cute.select(mV.layout, mode=V_layout_transpose))++ self.q_major_mode = cutlass.utils.LayoutEnum.from_tensor(mQ).mma_major_mode()+ self.k_major_mode = cutlass.utils.LayoutEnum.from_tensor(mK).mma_major_mode()+ self.v_major_mode = cutlass.utils.LayoutEnum.from_tensor(mV).mma_major_mode()+ self.o_layout = cutlass.utils.LayoutEnum.from_tensor(mO)++ if const_expr(self.q_major_mode != tcgen05.OperandMajorMode.K):+ raise RuntimeError("The layout of mQ is not supported")+ if const_expr(self.k_major_mode != tcgen05.OperandMajorMode.K):+ raise RuntimeError("The layout of mK is not supported")+ # FP4 block-scaled MMA requires K-major B; standard MMA uses MN-major+ if const_expr(mSFV is not None):+ if const_expr(self.v_major_mode != tcgen05.OperandMajorMode.K):+ raise RuntimeError("The layout of mV must be K-major for FP4 block-scaled MMA")+ else:+ if const_expr(self.v_major_mode != tcgen05.OperandMajorMode.MN):+ raise RuntimeError("The layout of mV is not supported")++ # check type consistency+ if const_expr(self.q_dtype == cutlass.Int8):+ assert self.q_dtype == self.k_dtype+ self.q_dtype = self.k_dtype = cutlass.Float4E2M1FN+ if const_expr(mSFV is not None):+ self.v_dtype = cutlass.Float4E2M1FN++ if const_expr(self.q_dtype != self.k_dtype):+ raise TypeError(f"Type mismatch: {self.q_dtype} != {self.k_dtype}")+ if const_expr(mSFV is not None and self.q_dtype != self.v_dtype):+ raise TypeError(f"Type mismatch: {self.q_dtype} != {self.v_dtype} (V quantization requires matching dtype)")+ self._setup_attributes()+ self.use_tma_O = (+ self.arch >= 90+ and mCuSeqlensQ is None+ and mSeqUsedQ is None+ and not self.seqlen_q_static_one+ )+ # This can be tuned+ self.e2e_freq = 16+ if const_expr(+ self.head_dim_padded > 64 and not self.is_causal and not self.is_local and self.pack_gqa+ ):+ self.e2e_freq = 32 if mCuSeqlensQ is not None or mSeqUsedQ is not None else 10++ use_2cta_instrs = self.mma_tiler_qk[0] == 256+ assert use_2cta_instrs == False, "Two-CTA instructions not supported yet"+ self.cta_group = (+ tcgen05.CtaGroup.TWO if use_2cta_instrs else tcgen05.CtaGroup.ONE+ )+ # the intermediate tensor p is from tmem & mK-major. Transposed, softmax+ # holds P in registers rather than tensor memory, and OperandSource only+ # selects the source of A, so P has to travel through shared memory.+ p_source = (+ tcgen05.OperandSource.SMEM+ if const_expr(self.transpose_s)+ else tcgen05.OperandSource.TMEM+ )+ p_major_mode = tcgen05.OperandMajorMode.K++ # Transposing S makes K the A operand and Q the B operand; both are+ # K-major over the head dimension either way, so only the roles move.+ qk_a_major = self.k_major_mode if const_expr(self.transpose_s) else self.q_major_mode+ qk_b_major = self.q_major_mode if const_expr(self.transpose_s) else self.k_major_mode++ # Use block-scaled MMA for PV only if V is being quantized (mSFV is provided)+ if const_expr(self.quant_qk):+ tiled_mma_qk = sm100_utils_basic.make_blockscaled_trivial_tiled_mma(+ self.q_dtype,+ qk_a_major,+ qk_b_major,+ self.sf_dtype,+ self.sf_vec_size,+ self.cta_group,+ self.mma_tiler_qk[:2],+ )+ else:+ tiled_mma_qk = sm100_utils_basic.make_trivial_tiled_mma(+ self.q_dtype,+ qk_a_major,+ qk_b_major,+ self.qk_acc_dtype,+ self.cta_group,+ self.mma_tiler_qk[:2],+ )++ if const_expr(self.quant_pv):+ tiled_mma_pv = sm100_utils_basic.make_blockscaled_trivial_tiled_mma(+ self.v_dtype,+ p_major_mode,+ self.v_major_mode,+ self.sf_dtype,+ self.sf_vec_size,+ self.cta_group,+ self.mma_tiler_pv[:2],+ p_source,+ )+ else:+ tiled_mma_pv = sm100_utils_basic.make_trivial_tiled_mma(+ self.v_dtype,+ p_major_mode,+ self.v_major_mode,+ self.pv_acc_dtype,+ self.cta_group,+ self.mma_tiler_pv[:2],+ p_source,+ )++ self.cluster_shape_mnk = (*self.cluster_shape_mn, 1)+ self.cluster_layout_vmnk = cute.tiled_divide(+ cute.make_layout(self.cluster_shape_mnk),+ (tiled_mma_qk.thr_id.shape,),+ )++ self.epi_tile = self.mma_tiler_pv[:2]++ # Which builder a tensor uses follows its operand role, not its name.+ make_q_smem_layout = (+ sm100_utils_basic.make_smem_layout_b+ if const_expr(self.transpose_s)+ else sm100_utils_basic.make_smem_layout_a+ )+ make_k_smem_layout = (+ sm100_utils_basic.make_smem_layout_a+ if const_expr(self.transpose_s)+ else sm100_utils_basic.make_smem_layout_b+ )+ make_sfq_smem_layout = (+ make_smem_layout_sfb if const_expr(self.transpose_s) else make_smem_layout_sfa+ )+ make_sfk_smem_layout = (+ make_smem_layout_sfa if const_expr(self.transpose_s) else make_smem_layout_sfb+ )++ # ((Atom_Inst_M, Atom_Inst_K), MMA_M, MMA_K, STAGE)+ sQ_layout = make_q_smem_layout(+ tiled_mma_qk,+ self.mma_tiler_qk,+ self.q_dtype,+ self.q_stage,+ )+ sK_layout = make_k_smem_layout(+ tiled_mma_qk,+ self.mma_tiler_qk,+ self.k_dtype,+ self.kv_stage,+ )+ tP_layout = sm100_utils_basic.make_smem_layout_a(+ tiled_mma_pv,+ self.mma_tiler_pv,+ self.v_dtype,+ self.acc_stage,+ )+ sV_layout = sm100_utils_basic.make_smem_layout_b(+ tiled_mma_pv,+ self.mma_tiler_pv,+ self.v_dtype,+ self.kv_stage,+ )+ sO_layout = sm100_utils_basic.make_smem_layout_epi(+ self.o_dtype,+ self.o_layout,+ self.epi_tile,+ self.epi_stage,+ )++ sfv_smem_layout_staged = None+ sfp_smem_layout_staged = None+ # # (((Atom_Inst_M, Rest_M),(Atom_Inst_K, Rest_K)), MMA_M, MMA_K, STAGE)+ sfq_smem_layout_staged = make_sfq_smem_layout(+ tiled_mma_qk,+ self.mma_tiler_qk,+ self.sf_vec_size,+ self.q_stage,+ mma_tile_inst_k=self.mma_inst_tile_k,+ )+ sfk_smem_layout_staged = make_sfk_smem_layout(+ tiled_mma_qk,+ self.mma_tiler_qk,+ self.sf_vec_size,+ self.kv_stage,+ mma_tile_inst_k=self.mma_inst_tile_k,+ )+ # Create P scale factor layout for P*V operation (P is the A matrix)+ if const_expr(self.quant_pv):+ sfp_smem_layout_staged = make_smem_layout_sfa(+ tiled_mma_pv,+ self.mma_tiler_pv,+ self.sf_vec_size,+ self.q_stage,+ mma_tile_inst_k=self.mma_inst_tile_k,+ )++ sfv_smem_layout_staged = make_smem_layout_sfb(+ tiled_mma_pv,+ self.mma_tiler_pv,+ self.sf_vec_size,+ self.kv_stage,+ mma_tile_inst_k=self.mma_inst_tile_k,+ )++ if const_expr(not self.same_hdim_kv_padded):+ # sK and sV are using the same physical smem so we need to adjust the stride so that they line up+ stride_sK = const_expr(+ max(sK_layout.outer.stride[-1], 0)+ ) # take max to turn tuple to Int32+ stride_sV = const_expr(max(sV_layout.outer.stride[-1], 0))+ stage_stride = const_expr(+ max(stride_sK, stride_sV)+ if not self.uneven_kv_smem+ else (stride_sK + stride_sV) // 2+ )+ sK_layout = cute.make_composed_layout(+ sK_layout.inner,+ 0,+ cute.make_layout(+ (*sK_layout.outer.shape[:-1], self.kv_stage),+ stride=(*sK_layout.outer.stride[:-1], stage_stride),+ ),+ )+ sV_layout = cute.make_composed_layout(+ sV_layout.inner,+ 0,+ cute.make_layout(+ (*sV_layout.outer.shape[:-1], self.kv_stage),+ stride=(*sV_layout.outer.stride[:-1], stage_stride),+ ),+ )+++ mQ_shape_unpacked = mQ.shape++ def packed_rows(t, head_mode: int, heads: int):+ """Shape and stride of the packed row mode of a (sq, ..., h, ...) tensor.++ With a single query position the (head, seqlen) mode collapses to+ the head axis alone. A flat axis of ``heads`` rows tiles by the+ M tile for any ratio, with TMA zero-filling the rows past it; the+ hierarchical mode only tiles when the tile is a whole number of+ query groups.+ """+ if const_expr(self.seqlen_q_static_one):+ return heads, t.stride[head_mode]+ return (heads, t.shape[0]), (t.stride[head_mode], t.stride[0])++ if const_expr(self.pack_gqa):+ q_rows_shape, q_rows_stride = packed_rows(mQ, 2, self.qhead_per_kvhead)+ shape_Q_packed = (+ q_rows_shape, # qhead_per_kvhead, or (qhead_per_kvhead, sq)+ mQ.shape[1], # d+ mK.shape[2], # h_k+ *mQ.shape[3:], # b+ )+ stride_Q_packed = (+ q_rows_stride,+ mQ.stride[1],+ mQ.stride[2] * self.qhead_per_kvhead,+ *mQ.stride[3:],+ )+ mQ = cute.make_tensor(+ mQ.iterator, cute.make_layout(shape_Q_packed, stride=stride_Q_packed)+ )+ mQSecond = cute.make_tensor(+ mQSecond.iterator,+ cute.make_layout(shape_Q_packed, stride=stride_Q_packed),+ )+ o_rows_shape, o_rows_stride = packed_rows(mO, 2, self.qhead_per_kvhead_O)+ shape_O_packed = (+ o_rows_shape,+ mO.shape[1],+ mK.shape[2],+ *mO.shape[3:],+ )+ stride_O_packed = (+ o_rows_stride,+ mO.stride[1],+ mO.stride[2] * self.qhead_per_kvhead_O,+ *mO.stride[3:],+ )+ mO = cute.make_tensor(+ mO.iterator, cute.make_layout(shape_O_packed, stride=stride_O_packed)+ )+ if const_expr(mLSE is not None):+ lse_rows_shape, lse_rows_stride = packed_rows(+ mLSE, 1, self.qhead_per_kvhead+ )+ shape_LSE_packed = (+ lse_rows_shape,+ mK.shape[2],+ *mLSE.shape[2:],+ )+ stride_LSE_packed = (+ lse_rows_stride,+ mLSE.stride[1] * self.qhead_per_kvhead,+ *mLSE.stride[2:],+ )+ mLSE = cute.make_tensor(+ mLSE.iterator, cute.make_layout(shape_LSE_packed, stride=stride_LSE_packed)+ )++ self.tma_copy_bytes = {+ name: cute.size_in_bytes(mX.element_type, cute.select(layout, mode=[0, 1, 2]))+ for name, mX, layout in [+ ("Q", mQ, sQ_layout),+ ("K", mK, sK_layout),+ ("V", mV, sV_layout),+ ]+ }+ # Add scale factor copy bytes to Q/K/V since they use the same barrier+ if const_expr(self.quant_qk):+ self.tma_copy_bytes["Q"] += cute.size_in_bytes(mSFQ.element_type, cute.select(sfq_smem_layout_staged, mode=[0, 1, 2]))+ self.tma_copy_bytes["K"] += cute.size_in_bytes(mSFK.element_type, cute.select(sfk_smem_layout_staged, mode=[0, 1, 2]))+ self.tma_copy_bytes["Q"] *= self.query_terms+ if const_expr(self.quant_pv):+ self.tma_copy_bytes["V"] += cute.size_in_bytes(mSFV.element_type, cute.select(sfv_smem_layout_staged, mode=[0, 1, 2]))++ # TMA load for Q+ tma_load_op = cpasync.CopyBulkTensorTileG2SOp(self.cta_group)+ tma_store_op = cpasync.CopyBulkTensorTileS2GOp()+ mQ_shape = mQ.shape+ # The FP4 Q tile is the B operand once S is transposed.+ make_q_tma_atom = (+ cute.nvgpu.make_tiled_tma_atom_B+ if const_expr(self.transpose_s)+ else cute.nvgpu.make_tiled_tma_atom_A+ )+ tma_atom_Q, mQ = make_q_tma_atom(+ tma_load_op,+ mQ,+ cute.select(sQ_layout, mode=[0, 1, 2]),+ self.mma_tiler_qk,+ tiled_mma_qk,+ self.cluster_layout_vmnk.shape,+ )+ tma_atom_Q_second, mQ_second_tma = make_q_tma_atom(+ tma_load_op,+ mQSecond,+ cute.select(sQ_layout, mode=[0, 1, 2]),+ self.mma_tiler_qk,+ tiled_mma_qk,+ self.cluster_layout_vmnk.shape,+ )++ tma_atom_K = None+ tma_atom_V = None+ mK_shape = mK.shape+ mV_shape = mV.shape+ if const_expr(self.use_tma_KV):+ # TMA load for K — the A operand once S is transposed.+ make_k_tma_atom = (+ cute.nvgpu.make_tiled_tma_atom_A+ if const_expr(self.transpose_s)+ else cute.nvgpu.make_tiled_tma_atom_B+ )+ tma_atom_K, mK = make_k_tma_atom(+ tma_load_op,+ mK,+ cute.select(sK_layout, mode=[0, 1, 2]),+ self.mma_tiler_qk,+ tiled_mma_qk,+ self.cluster_layout_vmnk.shape,+ )+ # TMA load for V+ tma_atom_V, mV = cute.nvgpu.make_tiled_tma_atom_B(+ tma_load_op,+ mV,+ cute.select(sV_layout, mode=[0, 1, 2]),+ self.mma_tiler_pv,+ tiled_mma_pv,+ self.cluster_layout_vmnk.shape,+ )++ tiled_mma_qk_bf16 = None+ tiled_mma_pv_bf16 = None+ sQ_bf16_layout = None+ sK_bf16_layout = None+ sV_bf16_layout = None+ tP_bf16_layout = None+ tma_atom_Q_bf16 = None+ tma_atom_K_bf16 = None+ tma_atom_V_bf16 = None+ mResidualQ_t = None+ mResidualK_t = None+ mResidualV_t = None+ if const_expr(self.fused_residual_first_block):+ if const_expr(self.residual_source == "contiguous"):+ mResK_src = mResidualK+ mResV_src = mResidualV+ else:+ # paged_bf16 — caller must supply both cache tensors.+ mResK_src = mResidualKCache+ mResV_src = mResidualVCache+ # mResK_src has logical shape (B-or-num_blocks, S=128, H_kv, D) bf16.+ # Apply the same transpose pattern as the BF16 reference kernel:+ # (b, s, h, d) -> (s, d, h, b) via mode=[1, 3, 2, 0].+ mResK = cute.make_tensor(+ mResK_src.iterator,+ cute.select(mResK_src.layout, mode=[1, 3, 2, 0]),+ )+ mResV = cute.make_tensor(+ mResV_src.iterator,+ cute.select(mResV_src.layout, mode=[1, 3, 2, 0]),+ )+ # BF16 V is MN-major (not transposed): (s, d, h, b) -> (d, s, h, b)+ mResV = cute.make_tensor(+ mResV.iterator,+ cute.select(mResV.layout, mode=[1, 0, 2, 3]),+ )+ # BF16 GEMMs use a non-blockscaled tiled MMA (standard SM100 BF16 tcgen05).+ # We re-use the existing q_major_mode/k_major_mode/v_major_mode+ # since the residual K/V are constructed with the same logical layout+ # transposes as the BF16 reference kernel.+ res_k_dtype = mResK.element_type # cutlass.BFloat16+ res_v_dtype = mResV.element_type # cutlass.BFloat16+ res_k_major = cutlass.utils.LayoutEnum.from_tensor(mResK).mma_major_mode()+ res_v_major = cutlass.utils.LayoutEnum.from_tensor(mResV).mma_major_mode()+ # The residual follows the FP4 blocks' orientation so that both write+ # the same tensor-memory tile the same way round and one softmax+ # reads them. Transposed, K is the A operand and Q the B operand.+ tiled_mma_qk_bf16 = sm100_utils_basic.make_trivial_tiled_mma(+ res_k_dtype,+ res_k_major if const_expr(self.transpose_s) else self.q_major_mode,+ self.q_major_mode if const_expr(self.transpose_s) else res_k_major,+ self.qk_acc_dtype,+ self.cta_group,+ self.mma_tiler_qk[:2],+ )+ tiled_mma_pv_bf16 = sm100_utils_basic.make_trivial_tiled_mma(+ res_v_dtype,+ p_major_mode,+ res_v_major,+ self.pv_acc_dtype,+ self.cta_group,+ self.mma_tiler_pv[:2],+ p_source,+ )+ # SMEM layouts for BF16 residual K/V (single stage — block 0 only).+ make_k_bf16_smem_layout = (+ sm100_utils_basic.make_smem_layout_a+ if const_expr(self.transpose_s)+ else sm100_utils_basic.make_smem_layout_b+ )+ sK_bf16_layout = make_k_bf16_smem_layout(+ tiled_mma_qk_bf16,+ self.mma_tiler_qk,+ res_k_dtype,+ 1, # 1 stage suffices: residual block 0 is loaded once per CTA-tile.+ )+ sV_bf16_layout = sm100_utils_basic.make_smem_layout_b(+ tiled_mma_pv_bf16,+ self.mma_tiler_pv,+ res_v_dtype,+ 1,+ )+ tP_bf16_layout = sm100_utils_basic.make_smem_layout_a(+ tiled_mma_pv_bf16,+ self.mma_tiler_pv,+ res_v_dtype, # BFloat16+ 1,+ )+ # TMA atoms for BF16 residual K/V.+ make_k_bf16_tma_atom = (+ cute.nvgpu.make_tiled_tma_atom_A+ if const_expr(self.transpose_s)+ else cute.nvgpu.make_tiled_tma_atom_B+ )+ tma_atom_K_bf16, mResidualK_t = make_k_bf16_tma_atom(+ tma_load_op,+ mResK,+ cute.select(sK_bf16_layout, mode=[0, 1, 2]),+ self.mma_tiler_qk,+ tiled_mma_qk_bf16,+ self.cluster_layout_vmnk.shape,+ )+ tma_atom_V_bf16, mResidualV_t = cute.nvgpu.make_tiled_tma_atom_B(+ tma_load_op,+ mResV,+ cute.select(sV_bf16_layout, mode=[0, 1, 2]),+ self.mma_tiler_pv,+ tiled_mma_pv_bf16,+ self.cluster_layout_vmnk.shape,+ )+ mResQ = cute.make_tensor(+ mResidualQ.iterator,+ cute.select(mResidualQ.layout, mode=[1, 3, 2, 0]),+ )+ if const_expr(self.pack_gqa):+ res_rows_shape, res_rows_stride = packed_rows(+ mResQ, 2, self.qhead_per_kvhead+ )+ shape_ResQ_packed = (+ res_rows_shape,+ mResQ.shape[1],+ mK.shape[2],+ *mResQ.shape[3:],+ )+ stride_ResQ_packed = (+ res_rows_stride,+ mResQ.stride[1],+ mResQ.stride[2] * self.qhead_per_kvhead,+ *mResQ.stride[3:],+ )+ mResQ = cute.make_tensor(+ mResQ.iterator,+ cute.make_layout(shape_ResQ_packed, stride=stride_ResQ_packed),+ )+ res_q_dtype = mResQ.element_type # cutlass.BFloat16+ make_q_bf16_smem_layout = (+ sm100_utils_basic.make_smem_layout_b+ if const_expr(self.transpose_s)+ else sm100_utils_basic.make_smem_layout_a+ )+ sQ_bf16_layout = make_q_bf16_smem_layout(+ tiled_mma_qk_bf16,+ self.mma_tiler_qk,+ res_q_dtype,+ 1,+ )+ make_q_bf16_tma_atom = (+ cute.nvgpu.make_tiled_tma_atom_B+ if const_expr(self.transpose_s)+ else cute.nvgpu.make_tiled_tma_atom_A+ )+ tma_atom_Q_bf16, mResidualQ_t = make_q_bf16_tma_atom(+ tma_load_op,+ mResQ,+ cute.select(sQ_bf16_layout, mode=[0, 1, 2]),+ self.mma_tiler_qk,+ tiled_mma_qk_bf16,+ self.cluster_layout_vmnk.shape,+ )++ # TMA load for scale factors+ tma_atom_sfq = None+ tma_tensor_sfq = None+ tma_atom_sfk = None+ tma_tensor_sfk = None+ tma_atom_sfv = None+ tma_tensor_sfv = None+ if const_expr(self.quant_qk):+ sfq_layout = cute.tile_to_shape(blockscaled_utils.BlockScaledBasicChunk(self.sf_vec_size).layout, mQ_shape_unpacked, (2, 1, 3, 4))+ sfq_op = sm100_utils_basic.cluster_shape_to_tma_atom_A(+ self.cluster_shape_mn, tiled_mma_qk.thr_id+ )+ mSFQ = cute.make_tensor(mSFQ.iterator, sfq_layout)+ mSFQSecond = cute.make_tensor(mSFQSecond.iterator, sfq_layout)+ if const_expr(self.pack_gqa):+ shape_SFQ_packed = (+ mSFQ.shape[0], # ((32,4), rest_m=1) unchanged+ mSFQ.shape[1], # K modes unchanged+ (mSFQ.shape[2][0], mK_shape[2]), # (1, h_kv=8) — use pre-TMA shape+ mSFQ.shape[3], # (1, b=1)+ )+ stride_SFQ_packed = (+ mSFQ.stride[0], # ((16,4), rest_m_stride) unchanged+ mSFQ.stride[1], # K strides unchanged+ mSFQ.stride[2], # (0, h_sf_stride=1024) unchanged+ mSFQ.stride[3], # batch unchanged+ )+ mSFQ = cute.make_tensor(+ mSFQ.iterator,+ cute.make_layout(shape_SFQ_packed, stride=stride_SFQ_packed),+ )+ mSFQSecond = cute.make_tensor(+ mSFQSecond.iterator,+ cute.make_layout(+ shape_SFQ_packed, stride=stride_SFQ_packed+ ),+ )+ tma_atom_sfq, tma_tensor_sfq = cute.nvgpu.make_tiled_tma_atom_A(+ sfq_op,+ mSFQ,+ cute.select(sfq_smem_layout_staged, mode=[0, 1, 2]),+ self.mma_tiler_qk,+ tiled_mma_qk,+ self.cluster_layout_vmnk.shape,+ internal_type=cutlass.Int16,+ )+ (+ tma_atom_sfq_second,+ tma_tensor_sfq_second,+ ) = cute.nvgpu.make_tiled_tma_atom_A(+ sfq_op,+ mSFQSecond,+ cute.select(sfq_smem_layout_staged, mode=[0, 1, 2]),+ self.mma_tiler_qk,+ tiled_mma_qk,+ self.cluster_layout_vmnk.shape,+ internal_type=cutlass.Int16,+ )++ if const_expr(self.quant_qk):+ # Setup TMA load for SFK (scale factor for K, like SFB).+ sfk_op = sm100_utils_basic.cluster_shape_to_tma_atom_SFB(+ self.cluster_shape_mn, tiled_mma_qk.thr_id+ )+ sfk_layout = cute.tile_to_shape(blockscaled_utils.BlockScaledBasicChunk(self.sf_vec_size).layout, mK_shape, (2, 1, 3, 4))+ sfk_layout = _sf_layout_with_page_stride(sfk_layout, k_sf_page_stride)+ mSFK = cute.make_tensor(mSFK.iterator, sfk_layout)++ # For SFB, compute mma_inst_shape_mnk_sfb: (M // (2 if use_2cta_instrs else 1), round_up(N, 128), K)+ mma_inst_shape_mnk_qk = (+ self.mma_tiler_qk[0],+ self.mma_tiler_qk[1],+ self.mma_inst_bits_k // self.k_dtype.width,+ )++ mma_inst_shape_mnk_sfb_qk = (+ mma_inst_shape_mnk_qk[0] // (2 if use_2cta_instrs else 1),+ cute.round_up(mma_inst_shape_mnk_qk[1], 128),+ mma_inst_shape_mnk_qk[2],+ )++ mma_tiler_sfb_qk = (+ mma_inst_shape_mnk_sfb_qk[0],+ mma_inst_shape_mnk_sfb_qk[1],+ mma_inst_shape_mnk_sfb_qk[2] * self.mma_inst_tile_k,+ )+ # For SFB, we need a separate tiled_mma_sfb with CtaGroup.ONE+ tiled_mma_sfb_qk = sm100_utils_basic.make_blockscaled_trivial_tiled_mma(+ self.k_dtype,+ self.k_major_mode,+ self.k_major_mode,+ self.sf_dtype,+ self.sf_vec_size,+ self.cta_group,+ mma_inst_shape_mnk_sfb_qk[:2],+ )+ cluster_layout_sfb_vmnk = cute.tiled_divide(+ cute.make_layout(self.cluster_shape_mnk),+ (tiled_mma_sfb_qk.thr_id.shape,),+ )+ tma_atom_sfk, tma_tensor_sfk = cute.nvgpu.make_tiled_tma_atom_B(+ sfk_op,+ mSFK,+ cute.select(sfk_smem_layout_staged, mode=[0, 1, 2]),+ mma_tiler_sfb_qk,+ tiled_mma_sfb_qk,+ cluster_layout_sfb_vmnk.shape,+ internal_type=cutlass.Int16,+ )++ if const_expr(self.quant_pv):+ # Setup TMA load for SFV (scale factor for V, like SFB)+ sfv_op = sm100_utils_basic.cluster_shape_to_tma_atom_SFB(+ self.cluster_shape_mn, tiled_mma_pv.thr_id+ )+ # Setup scale factor tensor layout+ sfv_layout = cute.tile_to_shape(blockscaled_utils.BlockScaledBasicChunk(self.sf_vec_size).layout, mV_shape, (2, 1, 3, 4))+ sfv_layout = _sf_layout_with_page_stride(sfv_layout, v_sf_page_stride)+ mSFV = cute.make_tensor(mSFV.iterator, sfv_layout)+ # For SFB, compute mma_inst_shape_mnk_sfb: (M // (2 if use_2cta_instrs else 1), round_up(N, 128), K)+ mma_inst_shape_mnk_pv = ( # the same processed by one tcgen05.mma instruction+ self.mma_tiler_pv[0],+ self.mma_tiler_pv[1],+ self.mma_inst_bits_k // self.v_dtype.width,+ )+ use_2cta_instrs = self.mma_tiler_pv[0] == 256+ mma_inst_shape_mnk_sfb_pv = (+ mma_inst_shape_mnk_pv[0] // (2 if use_2cta_instrs else 1),+ cute.round_up(mma_inst_shape_mnk_pv[1], 128),+ mma_inst_shape_mnk_pv[2],+ )+ mma_tiler_sfb_pv = (+ mma_inst_shape_mnk_sfb_pv[0],+ mma_inst_shape_mnk_sfb_pv[1],+ mma_inst_shape_mnk_sfb_pv[2] * self.mma_inst_tile_k,+ )+ # For SFB, we need a separate tiled_mma_sfb with CtaGroup.ONE+ tiled_mma_sfb_pv = sm100_utils_basic.make_blockscaled_trivial_tiled_mma(+ self.v_dtype,+ p_major_mode,+ self.v_major_mode,+ self.sf_dtype,+ self.sf_vec_size,+ self.cta_group,+ mma_inst_shape_mnk_sfb_pv[:2],+ p_source,+ )+ cluster_layout_sfb_pv_vmnk = cute.tiled_divide(+ cute.make_layout(self.cluster_shape_mnk),+ (tiled_mma_sfb_pv.thr_id.shape,),+ )+ tma_atom_sfv, tma_tensor_sfv = cute.nvgpu.make_tiled_tma_atom_B(+ sfv_op,+ mSFV,+ cute.select(sfv_smem_layout_staged, mode=[0, 1, 2]),+ mma_tiler_sfb_pv,+ tiled_mma_sfb_pv,+ cluster_layout_sfb_pv_vmnk.shape,+ internal_type=cutlass.Int16,+ )++ o_cta_v_layout = cute.composition(cute.make_identity_layout(mO.shape), self.epi_tile)++ self.num_epilogue_threads = cute.arch.WARP_SIZE * len(self.epilogue_warp_ids)+ if const_expr(self.use_tma_O):+ tma_atom_O, mO = cpasync.make_tiled_tma_atom(+ tma_store_op,+ mO,+ cute.select(sO_layout, mode=[0, 1]),+ o_cta_v_layout,+ )+ gmem_tiled_copy_O = None+ else:+ tma_atom_O = None+ universal_copy_bits = 128+ async_copy_elems = universal_copy_bits // self.o_dtype.width+ atom_universal_copy = cute.make_copy_atom(+ cute.nvgpu.CopyUniversalOp(),+ self.o_dtype,+ num_bits_per_copy=universal_copy_bits,+ )+ tO_shape_dim_1 = sO_layout.outer.shape[1][0] // async_copy_elems+ tO_layout = cute.make_ordered_layout(+ (self.num_epilogue_threads // tO_shape_dim_1, tO_shape_dim_1),+ order=(1, 0),+ )+ # So that we don't have to check if we overshoot kBlockM when we store O+ assert self.m_block_size % tO_layout.shape[0] == 0+ vO_layout = cute.make_layout((1, async_copy_elems))+ gmem_tiled_copy_O = cute.make_tiled_copy_tv(atom_universal_copy, tO_layout, vO_layout)++ if const_expr(mCuSeqlensQ is not None or mSeqUsedQ is not None):+ TileScheduler = SingleTileVarlenScheduler+ else:+ if const_expr(self.is_causal or self.is_local):+ TileScheduler = SingleTileLPTScheduler+ else:+ TileScheduler = (+ SingleTileScheduler+ if const_expr(not self.is_persistent)+ else StaticPersistentTileScheduler+ )+ tile_sched_args = TileSchedulerArguments(+ cute.ceil_div(cute.size(mQ.shape[0]), self.cta_tiler[0]),+ cute.size(mQ.shape[2]),+ cute.size(mQ.shape[3])+ if const_expr(mCuSeqlensQ is None)+ else cute.size(mCuSeqlensQ.shape[0] - 1),+ num_splits,+ cute.size(mK.shape[0])+ if const_expr(mPageTable is None)+ else mK.shape[0] * mPageTable.shape[1],+ mQ.shape[1],+ mV.shape[0], # Note that this is different from Sm90 since we transpose mV in Sm100+ total_q=cute.size(mQ.shape[0])+ if const_expr(mCuSeqlensQ is not None)+ else cute.size(mQ.shape[0]) * cute.size(mQ.shape[3]),+ tile_shape_mn=self.cta_tiler[:2],+ mCuSeqlensQ=mCuSeqlensQ,+ mSeqUsedQ=mSeqUsedQ,+ qhead_per_kvhead_packgqa=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1,+ # For sub-byte dtypes (FP4), width=4 rounds to 0 under integer division;+ # clamp so size_one_head * element_size in the scheduler's L2 swizzle calc+ # doesn't become 0 and divide by zero in the else branch of the ifexp.+ element_size=max(self.k_dtype.width // 8, 1),+ is_persistent=self.is_persistent,+ lpt=self.is_causal or self.is_local,+ is_split_kv=self.is_split_kv,+ )+ tile_sched_params = TileScheduler.to_underlying_arguments(tile_sched_args)+ self.tile_scheduler_cls = TileScheduler+ grid_dim = TileScheduler.get_grid_shape(tile_sched_params)++ self.mbar_load_q_full_offset = 0+ self.mbar_load_q_empty_offset = self.mbar_load_q_full_offset + self.q_stage+ self.mbar_load_kv_full_offset = self.mbar_load_q_empty_offset + self.q_stage+ self.mbar_load_kv_empty_offset = self.mbar_load_kv_full_offset + self.kv_stage+ self.mbar_P_full_O_rescaled_offset = self.mbar_load_kv_empty_offset + self.kv_stage+ self.mbar_S_full_offset = self.mbar_P_full_O_rescaled_offset + self.q_stage+ self.mbar_O_full_offset = self.mbar_S_full_offset + self.q_stage+ self.mbar_softmax_corr_full_offset = self.mbar_O_full_offset + self.q_stage+ self.mbar_softmax_corr_empty_offset = self.mbar_softmax_corr_full_offset + self.q_stage+ self.mbar_corr_epi_full_offset = self.mbar_softmax_corr_empty_offset + self.epi_stage+ self.mbar_corr_epi_empty_offset = self.mbar_corr_epi_full_offset + self.epi_stage+ self.mbar_s0_s1_sequence_offset = self.mbar_corr_epi_empty_offset + self.q_stage+ self.mbar_tmem_dealloc_offset = self.mbar_s0_s1_sequence_offset + 8+ self.mbar_P_full_2_offset = self.mbar_tmem_dealloc_offset + 1+ # QK and PV SF tmem load wait for softmax t2r store+ self.mbar_sfqk_load_offset = self.mbar_P_full_2_offset + self.q_stage+ # The PV GEMM has read the shared-memory P and its scale factors.+ self.mbar_P_consumed_offset = self.mbar_sfqk_load_offset + self.q_stage+ self.mbar_q_fp4_ready_offset = self.mbar_P_consumed_offset + self.q_stage+ self.mbar_residual_kv_full_offset = self.mbar_q_fp4_ready_offset + self.q_stage+ # `self.fused_residual_first_block` is a constant (set in __init__) so+ # the branch is selected at trace time; no const_expr wrapper needed+ # since these are plain Python int assignments.+ _residual_kv_slots = 3 if self.fused_residual_first_block else 0+ self.mbar_residual_kv_empty_offset = self.mbar_residual_kv_full_offset + _residual_kv_slots+ self.mbar_bf16_S_full_offset = self.mbar_residual_kv_empty_offset + _residual_kv_slots+ _bf16_phase_slots = 3 if self.fused_residual_first_block else 0+ self.mbar_bf16_P_full_offset = self.mbar_bf16_S_full_offset + (1 if self.fused_residual_first_block else 0)+ self.mbar_bf16_P_full_2_offset = self.mbar_bf16_P_full_offset + (1 if self.fused_residual_first_block else 0)+ self.mbar_bf16_PV_done_offset = self.mbar_bf16_P_full_2_offset + (1 if self.fused_residual_first_block else 0)+ self.mbar_total = self.mbar_bf16_PV_done_offset + (1 if self.fused_sink_block else 0)+ # self.mbar_total = self.mbar_P_full_2_offset + self.q_stage+ self.mbar_p_split = lambda k: (k // 4 * 3 if cutlass.const_expr(self.v_dtype.width >= 8) else k // 2 )+ sO_size = (+ cute.cosize(sO_layout)+ if const_expr(not (self.overlap_sO_sQ or self.overlap_sO_bf16))+ else 1+ )+ # The query terms and their scale factors live in sQ only while QK+ # runs and sO only after the last MMA, so an overlapped region holds+ # the larger of the two, not one O tile per term. The scale factors+ # follow the Q tiles there, which leaves sSFQ a placeholder.+ q_terms_size = self.query_terms * cute.cosize(sQ_layout)+ sfq_terms_size = (+ self.query_terms * cute.cosize(sfq_smem_layout_staged)+ if const_expr(self.quant_qk)+ else 0+ )+ sQ_size = (+ cutlass.max(+ q_terms_size + sfq_terms_size * 8 // self.q_dtype.width,+ cute.cosize(sO_layout) * self.o_dtype.width // self.q_dtype.width,+ )+ if const_expr(self.overlap_sO_sQ)+ else q_terms_size+ )++ # Calculate scale factor shared memory sizes+ # Use size 1 as minimum to avoid alignment issues when size is 0+ sfq_smem_size = (+ sfq_terms_size+ if const_expr(self.quant_qk and not self.overlap_sO_sQ)+ else 1+ )+ sfk_smem_size = cute.cosize(sfk_smem_layout_staged) if const_expr(self.quant_qk) else 1+ sfp_smem_size = cute.cosize(sfp_smem_layout_staged) if const_expr(self.quant_pv) else 1+ sfv_smem_size = cute.cosize(sfv_smem_layout_staged) if const_expr(self.quant_pv) else 1++ # P leaves tensor memory under the transposed layout, so the same tile+ # the MMA reads as its A operand needs real storage. sSoftmaxRed carries+ # the per-warp partial row maxima between the four softmax warps.+ sP_smem_size = cute.cosize(tP_layout) if const_expr(self.transpose_s) else 1+ softmax_red_size = (+ self.softmax_red_slots if const_expr(self.transpose_s) else 1+ )++ sK_bf16_smem_size = (+ cute.cosize(sK_bf16_layout) if const_expr(self.fused_residual_first_block) else 1+ )+ sV_bf16_smem_size = (+ cute.cosize(sV_bf16_layout) if const_expr(self.fused_residual_first_block) else 1+ )+ # Transposed, the residual's P also reaches the tensor core from shared+ # memory. It is the same 128x128 of bf16 as the residual Q tile and the+ # QK GEMM, whose completion softmax waits on before writing P, is+ # exactly when that tile dies, so the two share the bytes.+ sQ_bf16_smem_size = (+ max(cute.cosize(sQ_bf16_layout), cute.cosize(tP_bf16_layout))+ if const_expr(self.fused_residual_first_block)+ else 1+ )++ @cute.struct+ class BF16Tiles:+ sK_bf16: cute.struct.Align[+ cute.struct.MemRange[cutlass.BFloat16, sK_bf16_smem_size],+ self.buffer_align_bytes,+ ]+ sV_bf16: cute.struct.Align[+ cute.struct.MemRange[cutlass.BFloat16, sV_bf16_smem_size],+ self.buffer_align_bytes,+ ]+ sQ_bf16: cute.struct.Align[+ cute.struct.MemRange[cutlass.BFloat16, sQ_bf16_smem_size],+ self.buffer_align_bytes,+ ]++ # K reuses V's buffer when K is smaller (FP4 K in BF16 V), or same dtype+ k_aliases_v = self.k_dtype.width < self.v_dtype.width+ kv_stages_storage = cute.struct.Align[+ cute.struct.MemRange[self.k_dtype, 1] if const_expr(k_aliases_v) else cute.struct.MemRange[self.k_dtype, cute.cosize(sK_layout)],+ 1 if const_expr(k_aliases_v) else self.buffer_align_bytes,+ ]++ # A CTA on the split of the BF16 blocks holds no FP4 K/V stage and the+ # others hold no BF16 tile, so with that split the two share one+ # region in place of sK.+ @cute.union+ class KVStagesOrBF16Tiles:+ stages: kv_stages_storage+ bf16: BF16Tiles++ @cute.union+ class BF16TilesOrO:+ bf16: BF16Tiles+ o: cute.struct.Align[+ cute.struct.MemRange[self.o_dtype, cute.cosize(sO_layout)],+ self.buffer_align_bytes,+ ]++ o_storage = (+ BF16TilesOrO+ if const_expr(self.overlap_sO_bf16)+ else cute.struct.Align[+ cute.struct.MemRange[self.o_dtype, sO_size],+ self.buffer_align_bytes,+ ]+ )++ # Every block the tensor core reads its operands out of the K/V stages,+ # P and Q while TMA refills the stages. The softmax's cross-warp row+ # reduction goes through sSoftmaxRed on its critical path every block,+ # and in the same half of shared memory as that traffic its loads wait+ # behind it: with sSoftmaxRed, P and the stages in one half,+ # Qwen3-30B's merged decode ran 8% slower and Qwen3-4B's 3% slower.+ # So the streamed buffers come first, O (or the BF16 tiles), cold+ # through the main loop, carries what follows into the other half,+ # and sSoftmaxRed is last.+ @cute.struct+ class SharedStorage:+ sK: KVStagesOrBF16Tiles if const_expr(self.bf16_split) else kv_stages_storage+ sV: cute.struct.Align[+ cute.struct.MemRange[self.v_dtype, cute.cosize(sV_layout)] if not const_expr(self.v_dtype == self.k_dtype) else cute.struct.MemRange[self.k_dtype, 1],+ self.buffer_align_bytes if not const_expr(self.v_dtype == self.k_dtype) else 1,+ ]+ # Scale factor shared memory (if block-scaled quantization is used)+ sSFK: cute.struct.Align[+ cute.struct.MemRange[self.sf_dtype, sfk_smem_size],+ self.buffer_align_bytes,+ ]+ sSFV: cute.struct.Align[+ cute.struct.MemRange[self.sf_dtype, sfv_smem_size],+ self.buffer_align_bytes,+ ]+ sP: cute.struct.Align[+ cute.struct.MemRange[self.v_dtype, sP_smem_size],+ self.buffer_align_bytes if const_expr(self.transpose_s) else 1,+ ]+ sQ: cute.struct.Align[+ cute.struct.MemRange[self.q_dtype, sQ_size],+ self.buffer_align_bytes,+ ]+ sSFQ: cute.struct.Align[+ cute.struct.MemRange[self.sf_dtype, sfq_smem_size],+ self.buffer_align_bytes,+ ]+ sO: o_storage+ sScale: cute.struct.Align[cute.struct.MemRange[Float32, max(self.q_stage * self.m_block_size * 2, self.m_block_size * 4)], self.buffer_align_bytes]+ sSFP: cute.struct.Align[+ cute.struct.MemRange[self.sf_dtype, sfp_smem_size],+ self.buffer_align_bytes,+ ]+ # m_barriers for pipelines+ mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.mbar_total]+ # Tmem holding buffer+ tmem_holding_buf: Int32+ sSoftmaxRed: cute.struct.Align[+ cute.struct.MemRange[Float32, softmax_red_size],+ self.buffer_align_bytes if const_expr(self.transpose_s) else 1,+ ]++ # Remove scale factors to avoid OOM. Seems I can't set their size to 0+ @cute.struct+ class SharedStorageBF16:+ # m_barriers for pipelines+ mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.mbar_total]+ # Tmem holding buffer+ tmem_holding_buf: Int32+ sScale: cute.struct.Align[cute.struct.MemRange[Float32, max(self.q_stage * self.m_block_size * 2, self.m_block_size * 4)], self.buffer_align_bytes]+ sO: cute.struct.Align[+ cute.struct.MemRange[self.o_dtype, sO_size],+ self.buffer_align_bytes,+ ]+ sQ: cute.struct.Align[+ cute.struct.MemRange[self.q_dtype, sQ_size],+ self.buffer_align_bytes,+ ]+ sK: cute.struct.Align[+ # cute.cosize(sK_layout) is correct even in the case of self.uneven_kv_smem+ cute.struct.MemRange[self.k_dtype, cute.cosize(sK_layout)],+ self.buffer_align_bytes,+ ]+ self.shared_storage = SharedStorage if const_expr(self.quant_qk) or const_expr(self.quant_pv) else SharedStorageBF16++ # Verify shared memory fits within budget+ total_smem_bytes = self.shared_storage.size_in_bytes()+ assert total_smem_bytes <= 227 * 1024, (+ f"SharedStorage {total_smem_bytes // 1024}KB exceeds 227KB limit. "+ f"Reduce kv_stage (currently {self.kv_stage})."+ )+ print(f"Total shared memory used: {total_smem_bytes / 1024:.2f} KB")+ sO_bytes = (+ cute.size_in_bytes(self.o_dtype, sO_layout)+ if const_expr(not (self.overlap_sO_sQ or self.overlap_sO_bf16))+ else 0+ )+ sQ_bytes = cute.size_in_bytes(self.q_dtype, sQ_layout)+ sK_bytes = cute.size_in_bytes(self.k_dtype, sK_layout)+ sV_bytes = cute.size_in_bytes(self.v_dtype, sV_layout)+ sfq_bytes = cute.size_in_bytes(cutlass.Uint8, sfq_smem_layout_staged)+ sfk_bytes = cute.size_in_bytes(cutlass.Uint8, sfk_smem_layout_staged)+ sfp_bytes = cute.size_in_bytes(cutlass.Uint8, sfp_smem_layout_staged) if const_expr(sfp_smem_layout_staged is not None) else 0+ sfv_bytes = cute.size_in_bytes(cutlass.Uint8, sfv_smem_layout_staged) if const_expr(sfv_smem_layout_staged is not None) else 0+ print(f"sO_size: {sO_bytes / 1024:.2f} KB")+ print(f"sQ_size: {sQ_bytes / 1024:.2f} KB")+ print(f"sK_size: {sK_bytes / 1024:.2f} KB")+ if const_expr(self.v_dtype != self.k_dtype):+ print(f"sV_size: {sV_bytes / 1024:.2f} KB")+ print(f"sfq_smem_size: {sfq_bytes / 1024:.2f} KB")+ print(f"sfk_smem_size: {sfk_bytes / 1024:.2f} KB")+ print(f"sfp_smem_size: {sfp_bytes / 1024:.2f} KB")+ print(f"sfv_smem_size: {sfv_bytes / 1024:.2f} KB")++ LOG2_E = math.log2(math.e)+ if const_expr(self.score_mod is None):+ softmax_scale_log2 = softmax_scale * LOG2_E+ softmax_scale = None+ else:+ # NB: If a users passes in a score mod, we want to apply the score-mod in the sm_scaled qk+ # But in the original base 10. We hijack softmax_scale_log2 to just be the change of base+ # and correctly apply the softmax_scale prior to score_mod in the softmax step+ softmax_scale_log2 = LOG2_E+ softmax_scale = softmax_scale++ if const_expr(window_size_left is not None):+ window_size_left = Int32(window_size_left)+ if const_expr(window_size_right is not None):+ window_size_right = Int32(window_size_right)++ fastdiv_mods = None+ if cutlass.const_expr(aux_tensors is not None):+ seqlen_q = cute.size(mQ.shape[0]) // (+ self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1+ )+ seqlen_k = cute.size(mK.shape[0])+ seqlen_q_divmod = FastDivmod.create(seqlen_q)+ seqlen_k_divmod = FastDivmod.create(seqlen_k)+ fastdiv_mods = (seqlen_q_divmod, seqlen_k_divmod)++ self.use_block_sparsity = cutlass.const_expr(blocksparse_tensors is not None)+ if cutlass.const_expr(self.use_block_sparsity and mPageTable is not None):+ raise NotImplementedError("Block sparsity + paged KV not supported on SM100")++ # Launch the kernel synchronously+ self.kernel(+ mQ,+ mQ_second_tma,+ mK,+ mV,+ mO,+ mLSE,+ mCuSeqlensQ,+ mCuSeqlensK,+ mSeqUsedQ,+ mSeqUsedK,+ mPageTable,+ tma_atom_Q,+ tma_atom_Q_second,+ tma_atom_K,+ tma_atom_V,+ tma_atom_O,+ softmax_scale_log2,+ softmax_scale,+ window_size_left,+ window_size_right,+ learnable_sink,+ blocksparse_tensors,+ sQ_layout,+ sK_layout,+ tP_layout,+ sV_layout,+ sO_layout,+ gmem_tiled_copy_O,+ tiled_mma_qk,+ tiled_mma_pv,+ tile_sched_params,+ num_splits,+ aux_tensors,+ fastdiv_mods,+ tma_atom_sfq,+ tma_tensor_sfq,+ tma_atom_sfq_second,+ tma_tensor_sfq_second,+ tma_atom_sfk,+ tma_tensor_sfk,+ tma_atom_sfv,+ tma_tensor_sfv,+ sfq_smem_layout_staged,+ sfk_smem_layout_staged,+ sfp_smem_layout_staged,+ sfv_smem_layout_staged,+ sQ_bf16_layout,+ tiled_mma_qk_bf16,+ mResidualK_t,+ mResidualV_t,+ mResidualSeqUsedK,+ mSinkSeqUsedK,+ tma_atom_K_bf16,+ tma_atom_V_bf16,+ sK_bf16_layout,+ sV_bf16_layout,+ tiled_mma_pv_bf16,+ mResidualQ_t,+ tma_atom_Q_bf16,+ tP_bf16_layout,+ mResidualBlockIds,+ mOutIndices,+ ).launch(+ grid=grid_dim,+ block=[self.threads_per_cta, 1, 1],+ cluster=self.cluster_shape_mnk,+ smem=self.shared_storage.size_in_bytes(),+ stream=stream,+ min_blocks_per_mp=1,+ )++ # GPU device kernel+ @cute.kernel+ def kernel(+ self,+ mQ: cute.Tensor, # (s_q, d, h, b) or (total_q, d, h) if there is cu_seqlens_q+ mQSecond: cute.Tensor,+ mK: cute.Tensor, # (s_k, d, h_k, b_k) or (total_k, d, h_k) if there is cu_seqlens_k or (page_size, d, h_k, num_pages) if there is page_table+ mV: cute.Tensor, # (d, s_k, h_k, b_k) or (d, total_k, h_k) if there is cu_seqlens_k or (d, page_size, h_k, num_pages) if there is page_table+ mO: cute.Tensor,+ mLSE: Optional[cute.Tensor],+ mCuSeqlensQ: Optional[cute.Tensor],+ mCuSeqlensK: Optional[cute.Tensor],+ mSeqUsedQ: Optional[cute.Tensor],+ mSeqUsedK: Optional[cute.Tensor],+ mPageTable: Optional[cute.Tensor],+ tma_atom_Q: cute.CopyAtom,+ tma_atom_Q_second: cute.CopyAtom,+ tma_atom_K: Optional[cute.CopyAtom],+ tma_atom_V: Optional[cute.CopyAtom],+ tma_atom_O: Optional[cute.CopyAtom],+ softmax_scale_log2: Float32,+ softmax_scale: Float32 | None,+ window_size_left: Optional[Int32],+ window_size_right: Optional[Int32],+ learnable_sink: Optional[cute.Tensor],+ blocksparse_tensors: Optional[BlockSparseTensors],+ sQ_layout: cute.ComposedLayout,+ sK_layout: cute.ComposedLayout,+ tP_layout: cute.ComposedLayout,+ sV_layout: cute.ComposedLayout,+ sO_layout: cute.ComposedLayout,+ gmem_tiled_copy_O: Optional[cute.TiledCopy],+ tiled_mma_qk: cute.TiledMma,+ tiled_mma_pv: cute.TiledMma,+ tile_sched_params: ParamsBase,+ num_splits: Int32,+ aux_tensors: Optional[list] = None,+ fastdiv_mods=(None, None),+ tma_atom_sfq: Optional[cute.CopyAtom] = None,+ tma_tensor_sfq: Optional[cute.Tensor] = None,+ tma_atom_sfq_second: Optional[cute.CopyAtom] = None,+ tma_tensor_sfq_second: Optional[cute.Tensor] = None,+ tma_atom_sfk: Optional[cute.CopyAtom] = None,+ tma_tensor_sfk: Optional[cute.Tensor] = None,+ tma_atom_sfv: Optional[cute.CopyAtom] = None,+ tma_tensor_sfv: Optional[cute.Tensor] = None,+ sfq_smem_layout_staged: Optional[cute.Layout] = None,+ sfk_smem_layout_staged: Optional[cute.Layout] = None,+ sfp_smem_layout_staged: Optional[cute.Layout] = None,+ sfv_smem_layout_staged: Optional[cute.Layout] = None,+ sQ_bf16_layout: Optional[cute.ComposedLayout] = None,+ tiled_mma_qk_bf16: Optional[cute.TiledMma] = None,+ mResidualK: Optional[cute.Tensor] = None,+ mResidualV: Optional[cute.Tensor] = None,+ mResidualSeqUsedK: Optional[cute.Tensor] = None,+ mSinkSeqUsedK: Optional[cute.Tensor] = None,+ tma_atom_K_bf16: Optional[cute.CopyAtom] = None,+ tma_atom_V_bf16: Optional[cute.CopyAtom] = None,+ sK_bf16_layout: Optional[cute.ComposedLayout] = None,+ sV_bf16_layout: Optional[cute.ComposedLayout] = None,+ tiled_mma_pv_bf16: Optional[cute.TiledMma] = None,+ mResidualQ: Optional[cute.Tensor] = None,+ tma_atom_Q_bf16: Optional[cute.CopyAtom] = None,+ tP_bf16_layout: Optional[cute.ComposedLayout] = None,+ mResidualBlockIds: Optional[cute.Tensor] = None,+ mOutIndices: Optional[cute.Tensor] = None,+ ):+ """The device kernel implementation of the Fused Multi-Head Attention.++ This kernel coordinates multiple specialized warps to perform different phases of the FMHA computation:+ 1. Load warp: Loads Q, K, V data from global memory to shared memory using TMA+ 2. MMA warp: Performs matrix multiplications (Q*K^T and P*V)+ 3. Softmax warps: Compute softmax normalization on attention scores+ 4. Correction warps: Apply adjustments to intermediate results+ 5. Epilogue warp: Handles final output transformation and storage++ The kernel implements a complex pipeline with overlapping computation and memory operations,+ using tensor memory access (TMA) for efficient data loading, warp specialization for different+ computation phases, and optional attention masking.+ """++ warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx())+ iket.range_push("pro_init")+ # Prefetch tma descriptor+ if warp_idx == 0:+ cpasync.prefetch_descriptor(tma_atom_Q)+ if const_expr(tma_atom_K is not None):+ cpasync.prefetch_descriptor(tma_atom_K)+ if const_expr(tma_atom_V is not None):+ cpasync.prefetch_descriptor(tma_atom_V)+ if const_expr(tma_atom_O is not None):+ cpasync.prefetch_descriptor(tma_atom_O)+ if const_expr(tma_atom_sfq is not None):+ cpasync.prefetch_descriptor(tma_atom_sfq)+ if const_expr(tma_atom_sfk is not None):+ cpasync.prefetch_descriptor(tma_atom_sfk)+ if const_expr(tma_atom_sfv is not None):+ cpasync.prefetch_descriptor(tma_atom_sfv)++ # Alloc+ smem = cutlass.utils.SmemAllocator()+ storage = smem.allocate(self.shared_storage)++ mbar_ptr = storage.mbar_ptr.data_ptr()+ # Use the first N warps to initialize barriers+ if warp_idx == 1:+ # Init "full" barrier with number of producers, "empty" barrier with number of consumers+ for i in cutlass.range_constexpr(self.q_stage):+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_load_q_full_offset + i, 1+ )+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_load_q_empty_offset + i, len([self.mma_warp_id])+ )+ if warp_idx == 2:+ for i in cutlass.range_constexpr(self.q_stage):+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_softmax_corr_empty_offset + i,+ cute.arch.WARP_SIZE * len(self.correction_warp_ids),+ )+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_softmax_corr_full_offset + i,+ cute.arch.WARP_SIZE+ * len(self.softmax0_warp_ids)+ * self.softmax_row_groups,+ )+ if warp_idx == 3:+ if const_expr(self.s0_s1_barrier):+ for i in cutlass.range_constexpr(8):+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_s0_s1_sequence_offset + i, cute.arch.WARP_SIZE+ )+ if const_expr(not self.use_correction_warps_for_epi) and warp_idx == 4:+ for i in cutlass.range_constexpr(self.q_stage):+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_corr_epi_full_offset + i,+ cute.arch.WARP_SIZE * len(self.correction_warp_ids),+ )+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_corr_epi_empty_offset + i,+ cute.arch.WARP_SIZE * len(self.epilogue_warp_ids),+ )+ if warp_idx == 5:+ for i in cutlass.range_constexpr(self.q_stage):+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_P_full_O_rescaled_offset + i,+ cute.arch.WARP_SIZE+ * (+ len(self.softmax0_warp_ids) * self.softmax_row_groups+ + len(self.correction_warp_ids)+ ),+ )+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_S_full_offset + i, len([self.mma_warp_id])+ )+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_O_full_offset + i, len([self.mma_warp_id])+ )+ if warp_idx == 6:+ for i in cutlass.range_constexpr(self.q_stage):+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_P_full_2_offset + i,+ cute.arch.WARP_SIZE * len(self.softmax0_warp_ids),+ )+ if warp_idx == 7:+ # Every softmax warp in the CTA arrives here once it leaves its+ # role, whether or not it was handed a row group, and so does every+ # correction warp.+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_tmem_dealloc_offset,+ cute.arch.WARP_SIZE+ * (+ sum(len(ids) for ids in self.softmax_warp_ids)+ + len(self.correction_warp_ids)+ ),+ )+ if warp_idx == 8:+ for i in cutlass.range_constexpr(self.q_stage):+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_sfqk_load_offset + i,+ cute.arch.WARP_SIZE+ * len(self.softmax0_warp_ids)+ * self.softmax_row_groups,+ )+ if const_expr(self.fused_residual_first_block):+ for i in cutlass.range_constexpr(3):+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_residual_kv_full_offset + i, 1+ )+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_residual_kv_empty_offset + i,+ len([self.mma_warp_id]),+ )+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_bf16_S_full_offset, 1+ )+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_bf16_P_full_offset,+ cute.arch.WARP_SIZE+ * len(self.softmax0_warp_ids)+ * self.softmax_row_groups,+ )+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_bf16_P_full_2_offset,+ cute.arch.WARP_SIZE+ * len(self.softmax0_warp_ids)+ * self.softmax_row_groups,+ )+ if const_expr(self.fused_sink_block):+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_bf16_PV_done_offset, 1+ )+ if warp_idx == 9:+ for i in cutlass.range_constexpr(self.q_stage):+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_q_fp4_ready_offset + i,+ 1, # one arrive: the load warp (after cast)+ )+ cute.arch.mbarrier_init(+ mbar_ptr + self.mbar_P_consumed_offset + i, len([self.mma_warp_id])+ )+ iket.range_pop()+ # Relying on pipeline_kv constructor to call mbarrier_init_fence and sync+ iket.range_push("pro_sync")+ pipeline_kv = self.make_and_init_load_kv_pipeline(mbar_ptr + self.mbar_load_kv_full_offset)+ iket.range_pop()+ iket.range_push("pro_setup")++ # Generate smem tensor Q/K/V/O+ # (MMA, MMA_Q, MMA_D, PIPE)+ sQ = storage.sQ.get_tensor(sQ_layout.outer, swizzle=sQ_layout.inner)+ sQ_terms = tuple(+ cute.make_tensor(+ sQ.iterator + term * cute.cosize(sQ_layout),+ sQ.layout,+ )+ for term in range(self.query_terms)+ )+ sQ_bf16 = None+ sQ_uint8 = None+ # Byte view for moving complete FP4 rows: a thread owns whole bytes,+ # never a single nibble. Stacking and replication share this layout.+ if const_expr(self.q_replicate > 1 or self.stacked_query_terms):+ uint8_swizzle = cute.make_swizzle(2, 4, 3)+ # FP4 atom_K (per kH) = head_dim_padded//2 elements; Uint8+ # atom_K_byte = atom_K_FP4 // 2 = head_dim_padded // 4.+ atom_K_b_size = self.head_dim_padded // 4+ # M-stride in Uint8 bytes = head_dim_padded // 2 (one byte per 2+ # FP4 elements; one row spans head_dim_padded FP4 = head_dim/2 byte).+ row_bytes = self.head_dim_padded // 2+ kH_stride_bytes = atom_K_b_size # one kH atom = atom_K_b_size bytes+ stage_stride_bytes = self.m_block_size * row_bytes+ uint8_outer = cute.make_layout(+ ((self.m_block_size, atom_K_b_size), 1, 2, self.q_stage),+ stride=(((row_bytes, 1)), 0, kH_stride_bytes, stage_stride_bytes),+ )+ sQ_uint8 = cute.make_tensor(+ cute.recast_ptr(sQ.iterator, uint8_swizzle, cutlass.Uint8),+ uint8_outer,+ )+ # (MMA, MMA_K, MMA_D, PIPE)+ # K and V share physical SMEM:+ # - same dtype: V aliases K (V uses K's base pointer)+ # - FP4 K + BF16 V: K aliases V (K uses V's base pointer, stride scaled by dtype ratio)+ kv_stages = storage.sK.stages if const_expr(self.bf16_split) else storage.sK+ bf16_tiles = (+ storage.sK.bf16+ if const_expr(self.bf16_split)+ else (storage.sO.bf16 if const_expr(self.overlap_sO_bf16) else None)+ )+ if const_expr(self.v_dtype == self.k_dtype):+ sK = kv_stages.get_tensor(sK_layout.outer, swizzle=sK_layout.inner)+ sV = cute.make_tensor(cute.recast_ptr(sK.iterator, sV_layout.inner), sV_layout.outer)+ elif const_expr(self.k_dtype.width < self.v_dtype.width):+ # K aliases V's buffer — K is smaller, fits inside V's stage.+ # K's stage stride must match V's stage stride in bytes so they align.+ sV = storage.sV.get_tensor(sV_layout.outer, swizzle=sV_layout.inner)+ stride_sV = const_expr(max(sV_layout.outer.stride[-1], 0))+ stride_sK_aligned = const_expr(stride_sV * self.v_dtype.width // self.k_dtype.width)+ sK_outer_aligned = cute.make_layout(+ sK_layout.outer.shape,+ stride=(*sK_layout.outer.stride[:-1], stride_sK_aligned),+ )+ sK = storage.sV.get_tensor(sK_outer_aligned, swizzle=sK_layout.inner, dtype=self.k_dtype)+ else:+ sK = storage.sK.get_tensor(sK_layout.outer, swizzle=sK_layout.inner)+ sV = storage.sV.get_tensor(sV_layout.outer, swizzle=sV_layout.inner)++ if const_expr(self.overlap_sO_bf16):+ sO = storage.sO.o.get_tensor(sO_layout.outer, swizzle=sO_layout.inner)+ elif const_expr(not self.overlap_sO_sQ):+ sO = storage.sO.get_tensor(sO_layout.outer, swizzle=sO_layout.inner)+ else:+ sO = cute.make_tensor(cute.recast_ptr(sQ.iterator, sO_layout.inner, self.o_dtype), sO_layout.outer)++ sScale = storage.sScale.get_tensor(+ cute.make_layout(max(self.q_stage * self.m_block_size * 2, self.m_block_size * 4))+ )++ # Get scale factor shared memory tensors if they exist+ sSFV = None+ sSFP = None+ sSFQ = None+ sSFK = None+ if const_expr(self.quant_qk):+ if const_expr(self.overlap_sO_sQ):+ sSFQ = storage.sQ.get_tensor(+ sfq_smem_layout_staged, dtype=self.sf_dtype+ )+ q_terms_bytes = const_expr(+ self.query_terms * cute.cosize(sQ_layout) * self.q_dtype.width // 8+ )+ sSFQ = cute.make_tensor(sSFQ.iterator + q_terms_bytes, sSFQ.layout)+ else:+ sSFQ = storage.sSFQ.get_tensor(sfq_smem_layout_staged)+ sSFQ_terms = tuple(+ cute.make_tensor(+ sSFQ.iterator+ + term * cute.cosize(sfq_smem_layout_staged),+ sSFQ.layout,+ )+ for term in range(self.query_terms)+ )+ sSFK = storage.sSFK.get_tensor(sfk_smem_layout_staged)+ else:+ sSFQ_terms = (None,)+ if const_expr(sfp_smem_layout_staged is not None):+ sSFP = storage.sSFP.get_tensor(sfp_smem_layout_staged)+ if const_expr(sfv_smem_layout_staged is not None):+ sSFV = storage.sSFV.get_tensor(sfv_smem_layout_staged)++ sK_bf16 = None+ sV_bf16 = None+ sP_bf16 = None+ sP_bf16_mk = None+ if const_expr(self.fused_residual_first_block):+ sK_bf16 = bf16_tiles.sK_bf16.get_tensor(sK_bf16_layout.outer, swizzle=sK_bf16_layout.inner)+ sV_bf16 = bf16_tiles.sV_bf16.get_tensor(sV_bf16_layout.outer, swizzle=sV_bf16_layout.inner)+ sQ_bf16 = bf16_tiles.sQ_bf16.get_tensor(sQ_bf16_layout.outer, swizzle=sQ_bf16_layout.inner)+ if const_expr(self.transpose_s):+ sP_bf16 = bf16_tiles.sQ_bf16.get_tensor(+ tP_bf16_layout.outer, swizzle=tP_bf16_layout.inner+ )+ # The softmax writes one query row at a time down its own kv+ # column, so it wants the plain (query row, kv) view of the+ # operand layout rather than its atom decomposition.+ sP_bf16_mk = cute.composition(+ sP_bf16[None, None, None, 0],+ cute.make_layout((self.m_block_size, self.n_block_size)),+ )++ thr_mma_qk = tiled_mma_qk.get_slice(0) # default 1SM+ thr_mma_pv = tiled_mma_pv.get_slice(0) # default 1SM++ qk_acc_shape = thr_mma_qk.partition_shape_C(self.mma_tiler_qk[:2])+ tStS_fake = thr_mma_qk.make_fragment_C(qk_acc_shape)+++ tmem_ptr = cute.make_ptr(Float32, 0, mem_space=cute.AddressSpace.tmem, assumed_align=16)+ tStS = cute.make_tensor(tmem_ptr, tStS_fake.layout)++ pv_acc_shape = thr_mma_pv.partition_shape_C(self.mma_tiler_pv[:2])+ tOtO = thr_mma_pv.make_fragment_C(pv_acc_shape)++ tStSs = tuple(+ cute.make_tensor(tStS.iterator + self.tmem_s_offset[stage], tStS.layout)+ for stage in range(2)+ )+ tOtOs = tuple(+ cute.make_tensor(tOtO.iterator + self.tmem_o_offset[stage], tOtO.layout)+ for stage in range(self.q_stage)+ )++ # P in shared memory: the MMA A-operand tile, plus a byte view for the+ # softmax warps, which own single nibbles and can only store whole bytes.+ sP = None+ sP_uint8 = None+ if const_expr(self.transpose_s):+ sP = storage.sP.get_tensor(tP_layout.outer, swizzle=tP_layout.inner)+ sP_uint8 = cute.make_tensor(+ cute.recast_ptr(sP.iterator, cute.make_swizzle(2, 4, 3), cutlass.Uint8),+ cute.make_layout(+ (+ (self.m_block_size, self.n_block_size // 4),+ 1,+ 2,+ self.acc_stage,+ ),+ stride=(+ (self.n_block_size // 2, 1),+ 0,+ self.n_block_size // 4,+ self.m_block_size * self.n_block_size // 2,+ ),+ ),+ )+ sSoftmaxRed = storage.sSoftmaxRed.get_tensor(+ cute.make_layout(self.softmax_red_slots)+ )+ else:+ sSoftmaxRed = None++ if const_expr(self.transpose_s):+ tOrP = thr_mma_pv.make_fragment_A(sP)[None, None, None, 0]+ tOrPs = [tOrP, tOrP]+ else:+ tP = cute.make_tensor(tStS.iterator, tP_layout.outer)+ tOrP = thr_mma_pv.make_fragment_A(tP)[None, None, None, 0]++ tOrPs = [+ cute.make_tensor(+ tOrP.iterator+ + self.qk_acc_dtype.width // tOrP._dtype.width * self.tmem_p_offset[stage],+ tOrP.layout,+ )+ for stage in range(2)+ ]+ tOrPs_bf16 = (None, None)+ if const_expr(self.fused_residual_first_block):+ thr_mma_pv_bf16 = tiled_mma_pv_bf16.get_slice(0)+ if const_expr(self.transpose_s):+ tOrP_bf16 = thr_mma_pv_bf16.make_fragment_A(sP_bf16)[None, None, None, 0]+ if const_expr(self.fused_sink_block):+ # The sink occupies one 16-token BF16 MMA-K block. Keep the+ # original operand layout and expose only that first block.+ tOrP_sink_bf16 = cute.make_tensor(+ tOrP_bf16.iterator,+ cute.make_layout(+ (+ tOrP_bf16.layout.shape[0],+ tOrP_bf16.layout.shape[1],+ (1, 1),+ ),+ stride=tOrP_bf16.layout.stride,+ ),+ )+ tOrPs_bf16 = (tOrP_bf16, tOrP_sink_bf16)+ else:+ tOrPs_bf16 = (tOrP_bf16, tOrP_bf16)+ else:+ tP_bf16 = cute.make_tensor(tStS.iterator, tP_bf16_layout.outer)+ tOrP_bf16 = thr_mma_pv_bf16.make_fragment_A(tP_bf16)[None, None, None, 0]+ tOrPs_bf16 = tuple(+ cute.make_tensor(+ tOrP_bf16.iterator+ + self.qk_acc_dtype.width // tOrP_bf16._dtype.width+ * self.tmem_p_bf16_offset[stage],+ tOrP_bf16.layout,+ )+ for stage in range(2)+ )+ # Setup scale factor TMEM tensors and S2T copy operations+ # Use the TMEM region immediately following the accumulator (O tensor)++ # sf_tmem_ptr = cute.make_ptr(self.sf_dtype, 0, mem_space=cute.AddressSpace.tmem, assumed_align=16)++ align = 16 # Required for tcgen05.cp 4x32dp128bit.+ # find_tmem_tensor_col_offset returns u32 columns; convert to sf_dtype elements+ sf_dtype_per_u32 = 32 // self.sf_dtype.width+ tCtSFQ_terms = [[None] * self.q_stage]+ tCtSFKs = [None] * self.q_stage+ if const_expr(self.quant_qk):+ if const_expr(self.q_stage == 2):+ sfq_tmem_col_offsets = [self.tmem_s_offset[self.q_stage - 1 - stage] for stage in range(self.q_stage)]+ else:+ sfq_tmem_col_offsets = [self.tmem_s_offset[1]] # the unused stage-1 S region+ sfq_tmem_ptrs = [cute.recast_ptr(+ cute.make_ptr(Float32, sfq_tmem_col_offsets[stage], # shuffle to minimize dependency+ mem_space=cute.AddressSpace.tmem, assumed_align=align),+ dtype=self.sf_dtype) for stage in range(self.q_stage)+ ]++ # (MMA, MMA_M, MMA_K) — Q is the B operand once S is transposed.+ make_tmem_layout_sfq = (+ blockscaled_utils.make_tmem_layout_sfb+ if const_expr(self.transpose_s)+ else blockscaled_utils.make_tmem_layout_sfa+ )+ make_tmem_layout_sfk = (+ blockscaled_utils.make_tmem_layout_sfa+ if const_expr(self.transpose_s)+ else blockscaled_utils.make_tmem_layout_sfb+ )+ tCtSFQ_layout = make_tmem_layout_sfq(+ tiled_mma_qk,+ self.mma_tiler_qk,+ self.sf_vec_size,+ cute.slice_(sfq_smem_layout_staged, (None, None, None, 0)),+ )+ tCtSFQ_probe = cute.make_tensor(+ sfq_tmem_ptrs[0], tCtSFQ_layout+ )+ sfq_col_offset = tcgen05.find_tmem_tensor_col_offset(+ tCtSFQ_probe+ )+ sfq_offset = math.ceil(sfq_col_offset * sf_dtype_per_u32 / align) * align+ tCtSFQ_terms = [+ [+ cute.make_tensor(+ sfq_tmem_ptrs[stage] + term * sfq_offset,+ tCtSFQ_layout,+ )+ for stage in range(self.q_stage)+ ]+ for term in range(self.query_terms)+ ]++ # K scale factors follow every independently scaled query term.+ sfk_tmem_ptrs = [+ sfq_tmem_ptrs[stage] + self.query_terms * sfq_offset+ for stage in range(self.q_stage)+ ]++ # (MMA, MMA_N, MMA_K)+ tCtSFK_layout = make_tmem_layout_sfk(+ tiled_mma_qk,+ self.mma_tiler_qk,+ self.sf_vec_size,+ cute.slice_(sfk_smem_layout_staged, (None, None, None, 0)),+ )+ tCtSFKs = [cute.make_tensor(sfk_tmem_ptrs[stage], tCtSFK_layout) for stage in range(self.q_stage)]++ # Setup SFP and SFV TMEM tensors. These normally sit in the columns S+ # was read out of, which costs the next QK a wait on this block's PV.+ # Where tmem_sfpv_offset says there is room, they get their own columns+ # instead and the two GEMMs stop contending.+ tCtSFPs = [None] * self.q_stage+ tCtSFVs = [None] * self.q_stage+ if const_expr(self.quant_pv):+ if const_expr(self.tmem_sfpv_offset is not None):+ sfpv_col_offsets = [self.tmem_sfpv_offset] * self.q_stage+ else:+ sfpv_col_offsets = [self.tmem_s_offset[stage] for stage in range(self.q_stage)]+ sfp_tmem_ptrs = [cute.recast_ptr(+ cute.make_ptr(Float32, sfpv_col_offsets[stage],+ mem_space=cute.AddressSpace.tmem, assumed_align=align),+ dtype=self.sf_dtype) for stage in range(self.q_stage)]+ # A narrow PV swaps the operands, so P's scales are the B operand's+ # and V's the A operand's, as K's and Q's are on a transposed QK.+ # The shared-memory layouts are the same either way round for a+ # 128 x 128 tile; only the tensor-memory placement follows the role.+ make_tmem_layout_sfp = (+ blockscaled_utils.make_tmem_layout_sfb+ if const_expr(self.narrow_pv)+ else blockscaled_utils.make_tmem_layout_sfa+ )+ make_tmem_layout_sfv = (+ blockscaled_utils.make_tmem_layout_sfa+ if const_expr(self.narrow_pv)+ else blockscaled_utils.make_tmem_layout_sfb+ )+ # (MMA, MMA_M, MMA_K)+ tCtSFP_layout = make_tmem_layout_sfp(+ tiled_mma_pv,+ self.mma_tiler_pv,+ self.sf_vec_size,+ cute.slice_(sfp_smem_layout_staged, (None, None, None, 0)),+ )+ tCtSFPs = [cute.make_tensor(sfp_tmem_ptrs[stage], tCtSFP_layout) for stage in range(self.q_stage)]++ # Make SFV tmem tensor+ sfp_offset = math.ceil(tcgen05.find_tmem_tensor_col_offset(tCtSFPs[0]) * sf_dtype_per_u32 / align) * align+ sfv_tmem_ptrs = [sfp_tmem_ptrs[stage] + sfp_offset for stage in range(self.q_stage)]++ # (MMA, MMA_N, MMA_K) for P*V operation (V is the B matrix)+ tCtSFV_layout = make_tmem_layout_sfv(+ tiled_mma_pv,+ self.mma_tiler_pv,+ self.sf_vec_size,+ cute.slice_(sfv_smem_layout_staged, (None, None, None, 0)),+ )+ tCtSFVs = [cute.make_tensor(sfv_tmem_ptrs[stage], tCtSFV_layout) for stage in range(self.q_stage)]+ if const_expr(self.tmem_sfpv_offset is not None):+ sfpv_cols = math.ceil(+ (sfp_offset + tcgen05.find_tmem_tensor_col_offset(tCtSFVs[0]) * sf_dtype_per_u32)+ / sf_dtype_per_u32+ )+ assert self.tmem_sfpv_offset + sfpv_cols <= self.tmem_alloc_cols, (+ f"SFP/SFV need {sfpv_cols} columns at {self.tmem_sfpv_offset}, "+ f"past the {self.tmem_alloc_cols}-column capacity"+ )++ block_info = BlockInfo(+ # This is cta_tiler, not mma_tiler_qk, since we move by block by (2 * mma_tiler[0], mma_tiler[1])+ self.cta_tiler[0],+ self.cta_tiler[1],+ self.is_causal,+ self.is_local,+ self.is_split_kv,+ window_size_left,+ window_size_right,+ qhead_per_kvhead_packgqa=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1,+ )+ SeqlenInfoCls = partial(+ SeqlenInfoQK.create,+ seqlen_q_static=(+ 1+ if const_expr(self.seqlen_q_static_one)+ else mQ.shape[0] if const_expr(not self.pack_gqa) else mQ.shape[0][1]+ ),+ seqlen_k_static=mK.shape[0]+ if const_expr(mPageTable is None)+ else mK.shape[0] * mPageTable.shape[1],+ mCuSeqlensQ=mCuSeqlensQ,+ mCuSeqlensK=mCuSeqlensK,+ mSeqUsedQ=mSeqUsedQ,+ mSeqUsedK=mSeqUsedK,+ )+ AttentionMaskCls = partial(+ AttentionMask,+ self.m_block_size,+ self.n_block_size,+ window_size_left=window_size_left,+ window_size_right=window_size_right,+ qhead_per_kvhead_packgqa=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1,+ )+ TileSchedulerCls = partial(self.tile_scheduler_cls.create, tile_sched_params)+ is_bf16_cta = False+ if const_expr(self.bf16_split):+ is_bf16_cta = self.is_bf16_split(+ TileSchedulerCls().initial_work_tile_info().tile_idx[3], num_splits+ )+ iket.range_pop()++ # ///////////////////////////////////////////////////////////////////////////////+ # EMPTY+ # ///////////////////////////////////////////////////////////////////////////////+ if const_expr(len(self.empty_warp_ids) > 0):+ if warp_idx == self.empty_warp_ids[0]:+ cute.arch.warpgroup_reg_dealloc(self.num_regs_empty)++ if const_expr(len(self.empty_warp_ids) > 1):+ if warp_idx == self.empty_warp_ids[1]:+ cute.arch.warpgroup_reg_dealloc(self.num_regs_empty)++ assert len(self.empty_warp_ids) <= 2++ # ///////////////////////////////////////////////////////////////////////////////+ # LOAD+ # ///////////////////////////////////////////////////////////////////////////////+ if warp_idx >= self.load_warp_ids[0] and warp_idx <= self.load_warp_ids[-1]:+ role_lifetime = iket.range_start("load_life")+ cute.arch.warpgroup_reg_dealloc(self.num_regs_other)+ iket.range_push("load_main")+ self.run_split_role(is_bf16_cta, partial(+ self.load,+ thr_mma_qk,+ thr_mma_pv,+ mQ,+ mQSecond,+ mK,+ mV,+ sQ_terms,+ sK,+ sV,+ mPageTable,+ tma_atom_Q,+ tma_atom_Q_second,+ tma_atom_K,+ tma_atom_V,+ tma_atom_sfq,+ tma_tensor_sfq,+ tma_atom_sfq_second,+ tma_tensor_sfq_second,+ tma_atom_sfk,+ tma_tensor_sfk,+ sSFQ_terms,+ sSFK,+ pipeline_kv,+ mbar_ptr,+ block_info,+ num_splits,+ SeqlenInfoCls,+ TileSchedulerCls,+ blocksparse_tensors,+ tma_atom_sfv,+ tma_tensor_sfv,+ sfv_smem_layout_staged,+ sSFV,+ sQ_bf16=sQ_bf16,+ sQ_uint8=sQ_uint8,+ mResidualK_t=mResidualK,+ mResidualV_t=mResidualV,+ mResidualQ_t=mResidualQ,+ tma_atom_K_bf16=tma_atom_K_bf16,+ tma_atom_V_bf16=tma_atom_V_bf16,+ tma_atom_Q_bf16=tma_atom_Q_bf16,+ sK_bf16=sK_bf16,+ sV_bf16=sV_bf16,+ tiled_mma_qk_bf16=tiled_mma_qk_bf16,+ tiled_mma_pv_bf16=tiled_mma_pv_bf16,+ mResidualBlockIds=mResidualBlockIds,+ ))+ iket.range_pop()+ iket.range_end(role_lifetime)++ # ///////////////////////////////////////////////////////////////////////////////+ # MMA+ # ///////////////////////////////////////////////////////////////////////////////+ if warp_idx == self.mma_warp_id:+ role_lifetime = iket.range_start("mma_life")+ # if warp_idx == self.mma_warp_id or warp_idx == self.empty_warp_ids:+ cute.arch.warpgroup_reg_dealloc(self.num_regs_other)+ # Alloc tmem buffer+ tmem_alloc_cols = Int32(self.tmem_alloc_cols)+ if warp_idx == self.mma_warp_id:+ cute.arch.alloc_tmem(tmem_alloc_cols, storage.tmem_holding_buf)+ cute.arch.sync_warp()++ self.run_split_role(is_bf16_cta, partial(+ self.mma,+ tiled_mma_qk,+ tiled_mma_pv,+ sQ_terms,+ sK,+ sV,+ tStSs,+ tOtOs,+ tOrPs,+ pipeline_kv,+ mbar_ptr,+ block_info,+ num_splits,+ SeqlenInfoCls,+ TileSchedulerCls,+ blocksparse_tensors,+ sSFQ_terms,+ sSFK,+ sSFV,+ sSFP,+ tCtSFQ_terms,+ tCtSFKs,+ tCtSFPs,+ tCtSFVs,+ tiled_mma_qk_bf16=tiled_mma_qk_bf16,+ tiled_mma_pv_bf16=tiled_mma_pv_bf16,+ sK_bf16=sK_bf16,+ sV_bf16=sV_bf16,+ sQ_bf16=sQ_bf16,+ mResidualSeqUsedK=mResidualSeqUsedK,+ tOrPs_bf16=tOrPs_bf16,+ sP=sP,+ sP_bf16=sP_bf16,+ ))++ # if warp_idx == self.mma_warp_id:+ # dealloc tmem buffer+ cute.arch.relinquish_tmem_alloc_permit()+ cute.arch.mbarrier_wait(mbar_ptr + self.mbar_tmem_dealloc_offset, 0)+ tmem_alloc_cols = Int32(self.tmem_alloc_cols)+ # Retrieving tmem ptr and make acc+ tmem_ptr = cute.arch.retrieve_tmem_ptr(+ Float32,+ alignment=16,+ ptr_to_buffer_holding_addr=storage.tmem_holding_buf,+ )+ cute.arch.dealloc_tmem(tmem_ptr, tmem_alloc_cols)+ iket.range_end(role_lifetime)++ # ///////////////////////////////////////////////////////////////////////////////+ # Epilogue+ # ///////////////////////////////////////////////////////////////////////////////+ if const_expr(not self.use_correction_warps_for_epi):+ if warp_idx >= self.epilogue_warp_ids[0] and warp_idx <= self.epilogue_warp_ids[-1]:+ role_lifetime = iket.range_start("epi_life")+ cute.arch.warpgroup_reg_dealloc(self.num_regs_other)+ iket.range_push("epi_main")+ self.epilogue_s2g(+ mO,+ sO,+ gmem_tiled_copy_O,+ tma_atom_O,+ mbar_ptr,+ block_info,+ num_splits,+ SeqlenInfoCls,+ TileSchedulerCls,+ mOutIndices=mOutIndices,+ )+ iket.range_pop()+ iket.range_end(role_lifetime)++ # ///////////////////////////////////////////////////////////////////////////////+ # Softmax+ # ///////////////////////////////////////////////////////////////////////////////+ if warp_idx < self.correction_warp_ids[0]:+ role_lifetime = iket.range_start("softmax_life")+ # increase register after decreasing+ cute.arch.warpgroup_reg_alloc(self.num_regs_softmax)+ softmax_loop = partial(+ self.softmax_loop,+ softmax_scale_log2=softmax_scale_log2,+ softmax_scale=softmax_scale,+ thr_mma_qk=thr_mma_qk,+ sScale=sScale,+ mLSE=mLSE,+ learnable_sink=learnable_sink,+ mbar_ptr=mbar_ptr,+ block_info=block_info,+ num_splits=num_splits,+ SeqlenInfoCls=SeqlenInfoCls,+ AttentionMaskCls=AttentionMaskCls,+ TileSchedulerCls=TileSchedulerCls,+ aux_tensors=aux_tensors,+ fastdiv_mods=fastdiv_mods,+ blocksparse_tensors=blocksparse_tensors,+ sSFP=sSFP,+ mResidualSeqUsedK=mResidualSeqUsedK,+ mSinkSeqUsedK=mSinkSeqUsedK,+ sP_uint8=sP_uint8,+ sSoftmaxRed=sSoftmaxRed,+ sP_bf16=sP_bf16,+ sP_bf16_mk=sP_bf16_mk,+ )++ if const_expr(not self.s0_s1_barrier):+ stage = Int32(0 if warp_idx < self.softmax1_warp_ids[0] else 1)+ # Compute tCtSFP/tStSi unconditionally so they have stable types.+ # When q_stage=1 we always read [0]; when q_stage=2 we pick by stage.+ if const_expr(self.quant_pv):+ if const_expr(self.q_stage == 2):+ tCtSFP = tCtSFPs[0] if stage == 0 else tCtSFPs[1]+ else:+ tCtSFP = tCtSFPs[0]+ else:+ tCtSFP = None+ if const_expr(self.q_stage == 2):+ s_off = self.tmem_s_offset[0] if stage == 0 else self.tmem_s_offset[1]+ else:+ s_off = self.tmem_s_offset[0]+ tStSi_local = cute.make_tensor(tStS.iterator + s_off, tStS.layout)+ if const_expr(self.softmax_row_groups > 1):+ # Every group drives the same single Q stage over its own+ # query rows; specialising on the group keeps its row+ # offset, its scratch base and its barrier id compile-time+ # constants.+ softmax_warpgroup = warp_idx // len(self.softmax0_warp_ids)+ for group in cutlass.range_constexpr(self.softmax_row_groups):+ if softmax_warpgroup == group:+ self.run_split_role(is_bf16_cta, partial(+ softmax_loop,+ stage=0,+ tStSi=tStSi_local,+ tCtSFP=tCtSFP,+ row_group=group,+ ))+ elif const_expr(self.q_stage == 2) or stage == 0:+ self.run_split_role(is_bf16_cta, partial(+ softmax_loop,+ stage=stage,+ tStSi=tStSi_local,+ tCtSFP=tCtSFP, # need to copy P sf to tmem after exp+ ))+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_tmem_dealloc_offset)+ else:+ # If there's s0_s1_barrier, it's faster to have 2 WGs having different code+ if warp_idx < self.softmax1_warp_ids[0]:+ tStSi = cute.make_tensor(tStS.iterator + self.tmem_s_offset[0], tStS.layout)+ softmax_loop(stage=0, tStSi=tStSi, tCtSFP=tCtSFPs[0])+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_tmem_dealloc_offset)+ if warp_idx < self.correction_warp_ids[0] and warp_idx >= self.softmax1_warp_ids[0]:+ tStSi = cute.make_tensor(tStS.iterator + self.tmem_s_offset[1], tStS.layout)+ softmax_loop(stage=1, tStSi=tStSi, tCtSFP=tCtSFPs[1])+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_tmem_dealloc_offset)+ iket.range_end(role_lifetime)++ # ///////////////////////////////////////////////////////////////////////////////+ # Correction+ # ///////////////////////////////////////////////////////////////////////////////+ if warp_idx >= self.correction_warp_ids[0] and warp_idx < self.mma_warp_id:+ role_lifetime = iket.range_start("corr_life")+ cute.arch.warpgroup_reg_dealloc(self.num_regs_correction)+ iket.range_push("corr_main")+ self.run_split_role(is_bf16_cta, partial(+ self.correction_loop,+ thr_mma_qk,+ thr_mma_pv,+ tStS,+ tOtOs,+ sScale,+ mO,+ mLSE,+ sO,+ learnable_sink,+ gmem_tiled_copy_O,+ tma_atom_O,+ mbar_ptr,+ softmax_scale_log2,+ block_info,+ num_splits,+ SeqlenInfoCls,+ TileSchedulerCls,+ blocksparse_tensors,+ ))+ iket.range_pop()+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_tmem_dealloc_offset)+ iket.range_end(role_lifetime)++ return++ @cute.jit+ def load(+ self,+ thr_mma_qk: cute.ThrMma,+ thr_mma_pv: cute.ThrMma,+ mQ: cute.Tensor,+ mQSecond: cute.Tensor,+ mK: cute.Tensor,+ mV: cute.Tensor,+ sQ_terms: Tuple[cute.Tensor, ...],+ sK: cute.Tensor,+ sV: cute.Tensor,+ mPageTable: Optional[cute.Tensor],+ tma_atom_Q: cute.CopyAtom,+ tma_atom_Q_second: cute.CopyAtom,+ tma_atom_K: Optional[cute.CopyAtom],+ tma_atom_V: Optional[cute.CopyAtom],+ tma_atom_sfq: Optional[cute.CopyAtom],+ tma_tensor_sfq: Optional[cute.Tensor],+ tma_atom_sfq_second: Optional[cute.CopyAtom],+ tma_tensor_sfq_second: Optional[cute.Tensor],+ tma_atom_sfk: Optional[cute.CopyAtom],+ tma_tensor_sfk: Optional[cute.Tensor],+ sSFQ_terms: Tuple[Optional[cute.Tensor], ...],+ sSFK: Optional[cute.Tensor],+ pipeline_kv: cutlass.pipeline.PipelineAsync,+ mbar_ptr: cute.Pointer,+ block_info: BlockInfo,+ num_splits: Int32,+ SeqlenInfoCls: Callable,+ TileSchedulerCls: Callable,+ blocksparse_tensors: Optional[BlockSparseTensors],+ tma_atom_sfv: Optional[cute.CopyAtom] = None,+ tma_tensor_sfv: Optional[cute.Tensor] = None,+ sfv_smem_layout_staged: Optional[cute.ComposedLayout] = None,+ sSFV: Optional[cute.Tensor] = None,+ sQ_bf16: Optional[cute.Tensor] = None,+ sQ_uint8: Optional[cute.Tensor] = None,+ mResidualK_t: Optional[cute.Tensor] = None,+ mResidualV_t: Optional[cute.Tensor] = None,+ mResidualQ_t: Optional[cute.Tensor] = None,+ tma_atom_K_bf16: Optional[cute.CopyAtom] = None,+ tma_atom_V_bf16: Optional[cute.CopyAtom] = None,+ tma_atom_Q_bf16: Optional[cute.CopyAtom] = None,+ sK_bf16: Optional[cute.Tensor] = None,+ sV_bf16: Optional[cute.Tensor] = None,+ tiled_mma_qk_bf16: Optional[cute.TiledMma] = None,+ tiled_mma_pv_bf16: Optional[cute.TiledMma] = None,+ mResidualBlockIds: Optional[cute.Tensor] = None,+ role: cutlass.Constexpr = SplitRole.ALL,+ ):+ loads_pages = const_expr(role != SplitRole.BF16)+ loads_bf16 = const_expr(+ self.fused_residual_first_block and role != SplitRole.PAGES+ )+ num_load_threads = len(self.load_warp_ids) * cute.arch.WARP_SIZE+ tidx = cute.arch.thread_idx()[0] % num_load_threads+ q_producer_phase = Int32(1)+ kv_producer_state = cutlass.pipeline.make_pipeline_state(+ cutlass.pipeline.PipelineUserType.Producer, self.kv_stage+ )+ residual_kv_empty_producer_phase = Int32(1)+ residual_k_empty_producer_phase = Int32(1)+ residual_v_empty_producer_phase = Int32(1)+ residual_q_empty_producer_phase = Int32(1)+ tile_scheduler = TileSchedulerCls()+ work_tile = tile_scheduler.initial_work_tile_info()+ while work_tile.is_valid_tile:+ m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx+ seqlen = SeqlenInfoCls(batch_idx)+ # mQ: [s, d, h, b]+ mQ_cur = seqlen.offset_batch_Q(+ mQ, batch_idx, dim=3+ )[None, None, head_idx]+ gQ = cute.local_tile(+ mQ_cur,+ cute.select(self.mma_tiler_qk, mode=[0, 2]),+ (None, 0),+ )+ mQ_second_cur = seqlen.offset_batch_Q(+ mQSecond, batch_idx, dim=3+ )[None, None, head_idx]+ gQ_second = cute.local_tile(+ mQ_second_cur,+ cute.select(self.mma_tiler_qk, mode=[0, 2]),+ (None, 0),+ )++ head_idx_kv = (+ head_idx // self.qhead_per_kvhead if const_expr(not self.pack_gqa) else head_idx+ )+ if const_expr(mPageTable is None):+ if const_expr(not seqlen.has_cu_seqlens_k):+ mK_cur, mV_cur = [t[None, None, head_idx_kv, batch_idx] for t in (mK, mV)]+ else:+ mK_cur = cute.domain_offset((seqlen.offset_k, 0), mK[None, None, head_idx_kv])+ mV_cur = cute.domain_offset((0, seqlen.offset_k), mV[None, None, head_idx_kv])+ gK = cute.local_tile(mK_cur, cute.select(self.mma_tiler_qk, mode=[1, 2]), (None, 0))+ gV = cute.local_tile(mV_cur, cute.select(self.mma_tiler_pv, mode=[1, 2]), (0, None))+ else:+ # Need to keep batch coord None since we'll index into it with page idx+ mK_cur, mV_cur = [t[None, None, head_idx_kv, None] for t in (mK, mV)]+ gK = cute.local_tile(+ mK_cur, cute.select(self.mma_tiler_qk, mode=[1, 2]), (None, 0, None)+ )+ gV = cute.local_tile(+ mV_cur, cute.select(self.mma_tiler_pv, mode=[1, 2]), (0, None, None)+ )+ tSgQ = thr_mma_qk.partition_A(gQ)+ tSgQ_second = thr_mma_qk.partition_A(gQ_second)+ tSgK = thr_mma_qk.partition_B(gK)+ tOgV = thr_mma_pv.partition_B(gV)++ load_Q_fn, _, _ = copy_utils.tma_get_copy_fn(+ tma_atom_Q,+ 0,+ cute.make_layout(1),+ tSgQ,+ sQ_terms[0],+ )+ load_Q_second_fn = load_Q_fn+ if const_expr(self.query_terms == 2):+ (+ load_Q_second_fn,+ _,+ _,+ ) = copy_utils.tma_get_copy_fn(+ tma_atom_Q_second,+ 0,+ cute.make_layout(1),+ tSgQ_second,+ sQ_terms[1],+ )++ if const_expr(self.use_tma_KV):+ tKsK, tKgK = cpasync.tma_partition(+ tma_atom_K,+ 0, # no multicast+ cute.make_layout(1),+ cute.group_modes(sK, 0, 3),+ cute.group_modes(tSgK, 0, 3),+ )+ tVsV, tVgV = cpasync.tma_partition(+ tma_atom_V,+ 0, # no multicast+ cute.make_layout(1),+ cute.group_modes(sV, 0, 3),+ cute.group_modes(tOgV, 0, 3),+ )+ paged_kv_manager = None+ else:+ page_size = mK.shape[0]+ paged_kv_manager = PagedKVManager.create(+ mPageTable,+ mK,+ mV,+ FastDivmod.create(page_size),+ batch_idx,+ head_idx_kv,+ tidx,+ seqlen.seqlen_k,+ 0, # leftpad_k+ self.n_block_size,+ self.head_dim_padded,+ self.head_dim_v_padded,+ num_load_threads,+ mK.element_type,+ )+ tKsK, tKgK = None, None+ tVsV, tVgV = None, None++ load_SFQ_fn = None+ load_SFQ_second_fn = None+ if const_expr(self.quant_qk):+ tma_tensor_sfq_cur = seqlen.offset_batch_Q(+ tma_tensor_sfq, batch_idx, dim=3+ )[None, None, head_idx]+ gSFQ = cute.local_tile(+ tma_tensor_sfq_cur,+ cute.select(self.mma_tiler_qk, mode=[0, 2]),+ (None, 0),+ )+ tSgSFQ = thr_mma_qk.partition_A(gSFQ)+ load_SFQ_fn, _, _ = copy_utils.tma_get_copy_fn(+ tma_atom_sfq,+ 0,+ cute.make_layout(1),+ tSgSFQ,+ sSFQ_terms[0],+ filter_zeros=True,+ )+ load_SFQ_second_fn = load_SFQ_fn+ if const_expr(self.query_terms == 2):+ tma_tensor_sfq_second_cur = seqlen.offset_batch_Q(+ tma_tensor_sfq_second, batch_idx, dim=3+ )[None, None, head_idx]+ gSFQ_second = cute.local_tile(+ tma_tensor_sfq_second_cur,+ cute.select(self.mma_tiler_qk, mode=[0, 2]),+ (None, 0),+ )+ tSgSFQ_second = thr_mma_qk.partition_A(+ gSFQ_second+ )+ (+ load_SFQ_second_fn,+ _,+ _,+ ) = copy_utils.tma_get_copy_fn(+ tma_atom_sfq_second,+ 0,+ cute.make_layout(1),+ tSgSFQ_second,+ sSFQ_terms[1],+ filter_zeros=True,+ )++ # Partition SFK similar to K - index batch and head first like mK_cur+ tKsSFK = None+ tKgSFK = None+ if const_expr(self.quant_qk):+ if const_expr(mPageTable is None):+ if const_expr(not seqlen.has_cu_seqlens_k):+ tma_tensor_sfk_cur = tma_tensor_sfk[None, None, head_idx_kv, batch_idx]+ gSFK = cute.local_tile(tma_tensor_sfk_cur, cute.select(self.mma_tiler_qk, mode=[1, 2]), (None, 0))+ else:+ tma_tensor_sfk_cur = cute.domain_offset((seqlen.offset_k, 0), tma_tensor_sfk[None, None, head_idx_kv])+ gSFK = cute.local_tile(tma_tensor_sfk_cur, cute.select(self.mma_tiler_qk, mode=[1, 2]), (None, 0))+ else:+ tma_tensor_sfk_cur = tma_tensor_sfk[None, None, head_idx_kv, None]+ gSFK = cute.local_tile(tma_tensor_sfk_cur, cute.select(self.mma_tiler_qk, mode=[1, 2]), (None, 0, None))+ tSgSFK = thr_mma_qk.partition_B(gSFK)+ # Group only the first 3 modes (static MMA modes) to avoid grouping dynamic Rest modes+ tKsSFK, tKgSFK = cpasync.tma_partition(+ tma_atom_sfk,+ 0,+ cute.make_layout(1),+ cute.group_modes(sSFK, 0, 3),+ cute.group_modes(tSgSFK, 0, 3),+ )+ tKsSFK = cute.filter_zeros(tKsSFK)+ tKgSFK = cute.filter_zeros(tKgSFK)++ # Partition SFV similar to V+ tVsSFV, tVgSFV = None, None+ if const_expr(self.quant_pv):+ if const_expr(mPageTable is None):+ if const_expr(not seqlen.has_cu_seqlens_k):+ tma_tensor_sfv_cur = tma_tensor_sfv[None, None, head_idx_kv, batch_idx]+ gSFV = cute.local_tile(tma_tensor_sfv_cur, cute.select(self.mma_tiler_pv, mode=[1, 2]), (0, None))+ else:+ tma_tensor_sfv_cur = cute.domain_offset((0, seqlen.offset_k), tma_tensor_sfv[None, None, head_idx_kv])+ gSFV = cute.local_tile(tma_tensor_sfv_cur, cute.select(self.mma_tiler_pv, mode=[1, 2]), (0, None))+ else:+ tma_tensor_sfv_cur = tma_tensor_sfv[None, None, head_idx_kv, None]+ gSFV = cute.local_tile(tma_tensor_sfv_cur, cute.select(self.mma_tiler_pv, mode=[1, 2]), (0, None, None))+ tOgSFV = thr_mma_pv.partition_B(gSFV)+ # Group only the first 3 modes (static MMA modes) to avoid grouping dynamic Rest modes+ tVsSFV, tVgSFV = cpasync.tma_partition(+ tma_atom_sfv,+ 0,+ cute.make_layout(1),+ cute.group_modes(sSFV, 0, 3),+ cute.group_modes(tOgSFV, 0, 3),+ )+ tVsSFV = cute.filter_zeros(tVsSFV)+ tVgSFV = cute.filter_zeros(tVgSFV)++ load_Q = partial(+ self.load_Q,+ load_Q_fn,+ mbar_ptr + self.mbar_load_q_full_offset,+ mbar_ptr + self.mbar_load_q_empty_offset,+ phase=q_producer_phase,+ load_SFQ_fn=load_SFQ_fn,+ load_Q_second_fn=load_Q_second_fn,+ load_SFQ_second_fn=load_SFQ_second_fn,+ )+ replicate_Q = None+ if const_expr(self.q_replicate > 1):+ # Replication gets its own pass over the tile between the TMA+ # and the first MMA, signalling mbar_q_fp4_ready when the tile+ # is whole.+ replicate_Q = partial(+ self.replicate_Q_fp4_rows,+ sQ_uint8,+ sSFQ_terms[0],+ mbar_ptr + self.mbar_load_q_full_offset,+ mbar_ptr + self.mbar_q_fp4_ready_offset,+ phase=q_producer_phase ^ 1, # waits for the just-issued TMA+ tidx=tidx,+ )+ stack_Q = None+ if const_expr(self.stacked_query_terms):+ stack_Q = partial(+ self.stack_Q_fp4_terms,+ sQ_uint8,+ sQ_terms[1],+ sSFQ_terms[0],+ sSFQ_terms[1],+ mbar_ptr + self.mbar_load_q_full_offset,+ mbar_ptr + self.mbar_q_fp4_ready_offset,+ phase=q_producer_phase ^ 1,+ tidx=tidx,+ )+ # We have to use mbarrier directly in the load for KV instead of replying on+ # pipeline_kv, because we could have different number of TMA bytes for K and V+ load_K = partial(+ self.load_KV,+ tma_atom_K,+ tKgK,+ tKsK,+ paged_kv_manager,+ sK,+ mbar_ptr + self.mbar_load_kv_full_offset,+ mbar_ptr + self.mbar_load_kv_empty_offset,+ K_or_V="K",+ tma_atom_sf=tma_atom_sfk,+ tXgSF=tKgSFK,+ tXsSF=tKsSFK,+ )++ load_V = partial(+ self.load_KV,+ tma_atom_V,+ tVgV,+ tVsV,+ paged_kv_manager,+ sV,+ mbar_ptr + self.mbar_load_kv_full_offset,+ mbar_ptr + self.mbar_load_kv_empty_offset,+ K_or_V="V",+ tma_atom_sf=tma_atom_sfv,+ tXgSF=tVgSFV,+ tXsSF=tVsSFV,+ )++ if const_expr(not self.use_block_sparsity):+ iket.range_push("load_seqinfo")+ n_block_min, n_block_max = self.split_block_range(+ block_info, seqlen, m_block, split_idx, num_splits+ )+ iket.range_pop()+ has_work = self.tile_has_work(+ n_block_min, n_block_max, split_idx, num_splits+ )+ # A split with no kv block runs no MMA, so nothing would release+ # the BF16 tiles the sink reload waits on: load only for work.+ if has_work:+ if const_expr(loads_bf16):+ if const_expr(self.residual_source == "paged_bf16"):+ # Graph padding keeps row_to_slot=-1 so it cannot own or+ # write a tail. TMA still needs a legal source page before+ # the zero residual/sink lengths mask the row away.+ residual_kv_idx = cutlass.max(+ mResidualBlockIds[batch_idx], Int32(0)+ )+ else:+ residual_kv_idx = batch_idx+ # BF16 residual K/V layout: (s=128, d, h_kv, b-or-num_blocks).+ # Slice batch/block + head, local_tile to BLOCK_SIZE (=128),+ # partition via the BF16 tiled MMAs, then tma_partition with+ # the BF16 atoms.+ thr_mma_qk_bf16_slc = tiled_mma_qk_bf16.get_slice(0)+ thr_mma_pv_bf16_slc = tiled_mma_pv_bf16.get_slice(0)+ mResK_cur = mResidualK_t[None, None, head_idx_kv, residual_kv_idx]+ mResV_cur = mResidualV_t[None, None, head_idx_kv, residual_kv_idx]+ gResK = cute.local_tile(+ mResK_cur, cute.select(self.mma_tiler_qk, mode=[1, 2]), (None, 0)+ )+ gResV = cute.local_tile(+ mResV_cur, cute.select(self.mma_tiler_pv, mode=[1, 2]), (0, None)+ )+ tSgResK = thr_mma_qk_bf16_slc.partition_B(gResK)+ tOgResV = thr_mma_pv_bf16_slc.partition_B(gResV)+ tKsK_bf16, tKgResK = cpasync.tma_partition(+ tma_atom_K_bf16,+ 0, # no multicast+ cute.make_layout(1),+ cute.group_modes(sK_bf16, 0, 3),+ cute.group_modes(tSgResK, 0, 3),+ )+ tVsV_bf16, tVgResV = cpasync.tma_partition(+ tma_atom_V_bf16,+ 0, # no multicast+ cute.make_layout(1),+ cute.group_modes(sV_bf16, 0, 3),+ cute.group_modes(tOgResV, 0, 3),+ )+ mResQ_cur = mResidualQ_t[None, None, head_idx, batch_idx]+ gResQ = cute.local_tile(+ mResQ_cur, cute.select(self.mma_tiler_qk, mode=[0, 2]), (None, 0)+ )+ tSgResQ = thr_mma_qk_bf16_slc.partition_A(gResQ)+ tQsQ_bf16, tQgResQ = cpasync.tma_partition(+ tma_atom_Q_bf16,+ 0, # no multicast+ cute.make_layout(1),+ cute.group_modes(sQ_bf16, 0, 3),+ cute.group_modes(tSgResQ, 0, 3),+ )+ # BF16 K/V/Q are loaded into dedicated single-stage SMEM tiles+ # (sK_bf16 / sV_bf16 / sQ_bf16) using dedicated mbarriers+ # (mbar_residual_kv_full + 0 = K, +1 = V, +2 = Q). The FP4+ # mainloop's kv pipeline is unchanged. For subsequent work+ # tiles (M>1), wait on the empty barrier so the MMA warp+ # has finished consuming the previous tile's data before+ # the load warp overwrites the SMEM.+ bf16_full_mbar = mbar_ptr + self.mbar_residual_kv_full_offset+ bf16_empty_mbar = mbar_ptr + self.mbar_residual_kv_empty_offset+ if const_expr(self.use_tma_KV) or tidx < cute.arch.WARP_SIZE:+ iket.range_push("load_wait_res")+ cute.arch.mbarrier_wait(+ bf16_empty_mbar + 0,+ residual_k_empty_producer_phase+ if const_expr(self.fused_sink_block)+ else residual_kv_empty_producer_phase,+ )+ cute.arch.mbarrier_wait(+ bf16_empty_mbar + 1,+ residual_v_empty_producer_phase+ if const_expr(self.fused_sink_block)+ else residual_kv_empty_producer_phase,+ )+ cute.arch.mbarrier_wait(+ bf16_empty_mbar + 2,+ residual_q_empty_producer_phase+ if const_expr(self.fused_sink_block)+ else residual_kv_empty_producer_phase,+ )+ iket.range_pop()+ # Slot 0: BF16 K+ with cute.arch.elect_one():+ cute.arch.mbarrier_arrive_and_expect_tx(+ bf16_full_mbar + 0,+ cute.cosize(sK_bf16) * 2, # bf16 = 2 bytes+ )+ cute.copy(+ tma_atom_K_bf16,+ tKgResK[None, 0],+ tKsK_bf16[None, 0],+ tma_bar_ptr=bf16_full_mbar + 0,+ )+ # Slot 1: BF16 V+ with cute.arch.elect_one():+ cute.arch.mbarrier_arrive_and_expect_tx(+ bf16_full_mbar + 1,+ cute.cosize(sV_bf16) * 2,+ )+ cute.copy(+ tma_atom_V_bf16,+ tVgResV[None, 0],+ tVsV_bf16[None, 0],+ tma_bar_ptr=bf16_full_mbar + 1,+ )+ sQ_bf16_stage_elems = const_expr(cute.cosize(sQ_bf16))+ with cute.arch.elect_one():+ cute.arch.mbarrier_arrive_and_expect_tx(+ bf16_full_mbar + 2,+ sQ_bf16_stage_elems * 2,+ )+ cute.copy(+ tma_atom_Q_bf16,+ tQgResQ[None, m_block],+ tQsQ_bf16[None, 0],+ tma_bar_ptr=bf16_full_mbar + 2,+ )+ if const_expr(self.fused_sink_block):+ residual_k_empty_producer_phase ^= 1+ residual_v_empty_producer_phase ^= 1+ residual_q_empty_producer_phase ^= 1+ else:+ residual_kv_empty_producer_phase ^= 1+ if const_expr(loads_pages):+ if const_expr(self.use_tma_KV) or tidx < cute.arch.WARP_SIZE:+ load_Q(block=self.q_stage * m_block + 0, stage=0) # Q0 + SFQ0+ n_block_first = n_block_max - 1 if n_block_max > 0 else 0+ iket.range_push("load_pageidx")+ page_idx = (+ mPageTable[batch_idx, n_block_first]+ if const_expr(mPageTable is not None and self.use_tma_KV)+ else None+ )+ if const_expr(not self.use_tma_KV):+ paged_kv_manager.load_page_table(n_block_first)+ iket.range_pop()+ iket.range_push("load_issue_k0")+ load_K(block=n_block_max - 1, producer_state=kv_producer_state, page_idx=page_idx) # K0 + SFK0+ iket.range_pop()+ kv_producer_state.advance()+ if const_expr(self.q_stage == 2) and (const_expr(self.use_tma_KV) or tidx < cute.arch.WARP_SIZE):+ load_Q(block=self.q_stage * m_block + 1, stage=1) # Q1 + SFQ1+ if const_expr(self.q_replicate > 1):+ for qs in cutlass.range_constexpr(self.q_stage):+ replicate_Q(stage=qs)+ if const_expr(self.stacked_query_terms):+ # One warp owns both the byte copy and its single+ # ready-barrier arrival, including non-TMA KV loads.+ if tidx < cute.arch.WARP_SIZE:+ stack_Q(stage=0)+ q_producer_phase ^= 1+ load_V(block=n_block_max - 1, producer_state=kv_producer_state, page_idx=page_idx) # V0 + SFV0+ kv_producer_state.advance()+ for i in cutlass.range(n_block_max - 1 - n_block_min, unroll=1):+ n_block = n_block_max - 2 - i+ page_idx = (+ mPageTable[batch_idx, n_block]+ if const_expr(mPageTable is not None and self.use_tma_KV)+ else None+ )+ if const_expr(not self.use_tma_KV):+ paged_kv_manager.load_page_table(n_block)+ load_K(block=n_block, producer_state=kv_producer_state, page_idx=page_idx) # Ki + SFKi+ kv_producer_state.advance()+ load_V(block=n_block, producer_state=kv_producer_state, page_idx=page_idx) # Vi + SFVi+ kv_producer_state.advance()+ # The sink is merged after the last page, so its operands+ # are loaded behind the pages: they stream while the tail's+ # GEMMs run, and the sink's own load hides under the last+ # blocks. K can be replaced once the tail QK has issued; V+ # and Q/P stay resident until the tail PV consumes them, so+ # each goes through its own empty barrier.+ if const_expr(loads_bf16 and self.fused_sink_block):+ if const_expr(self.use_tma_KV) or tidx < cute.arch.WARP_SIZE:+ cute.arch.mbarrier_wait(+ bf16_empty_mbar + 0,+ residual_k_empty_producer_phase,+ )+ with cute.arch.elect_one():+ cute.arch.mbarrier_arrive_and_expect_tx(+ bf16_full_mbar + 0,+ cute.cosize(sK_bf16) * 2,+ )+ cute.copy(+ tma_atom_K_bf16,+ tKgResK[None, 1],+ tKsK_bf16[None, 0],+ tma_bar_ptr=bf16_full_mbar + 0,+ )+ residual_k_empty_producer_phase ^= 1++ cute.arch.mbarrier_wait(+ bf16_empty_mbar + 1,+ residual_v_empty_producer_phase,+ )+ with cute.arch.elect_one():+ cute.arch.mbarrier_arrive_and_expect_tx(+ bf16_full_mbar + 1,+ cute.cosize(sV_bf16) * 2,+ )+ cute.copy(+ tma_atom_V_bf16,+ tVgResV[None, 1],+ tVsV_bf16[None, 0],+ tma_bar_ptr=bf16_full_mbar + 1,+ )+ residual_v_empty_producer_phase ^= 1++ cute.arch.mbarrier_wait(+ bf16_empty_mbar + 2,+ residual_q_empty_producer_phase,+ )+ with cute.arch.elect_one():+ cute.arch.mbarrier_arrive_and_expect_tx(+ bf16_full_mbar + 2,+ cute.cosize(sQ_bf16) * 2,+ )+ cute.copy(+ tma_atom_Q_bf16,+ tQgResQ[None, m_block],+ tQsQ_bf16[None, 0],+ tma_bar_ptr=bf16_full_mbar + 2,+ )+ residual_q_empty_producer_phase ^= 1++ else:+ kv_producer_state, q_producer_phase = produce_block_sparse_loads_sm100(+ blocksparse_tensors,+ batch_idx,+ head_idx,+ m_block,+ kv_producer_state,+ load_Q,+ load_K,+ load_V,+ pipeline_kv,+ self.q_stage,+ q_producer_phase,+ )+++ tile_scheduler.prefetch_next_work()+ tile_scheduler.advance_to_next_work()+ work_tile = tile_scheduler.get_current_work()+ # End of persistent scheduler loop++ def qk_gemm(self, gemm, tCrK: cute.Tensor, sK_cur: cute.Tensor):+ """Issue one QK GEMM with the K tile in whichever operand slot it holds.++ K is the B operand normally and the A operand once S is transposed. The+ Q side is already bound into ``gemm``.+ """+ if const_expr(self.transpose_s):+ gemm(tCrA=tCrK, sA=sK_cur)+ else:+ gemm(tCrB=tCrK, sB=sK_cur)++ def release_shared_p(self, mbar_ptr: cute.Pointer, stage: int):+ """Signal once the PV GEMM just issued has read P from shared memory.++ Transposed, P and its scale factors sit in one shared-memory tile that+ the softmax rewrites for every block, and the next block's QK is issued+ ahead of this block's PV, so the S it signals says nothing about this+ PV. Every FP4 PV signals here, which keeps one completion per block.+ """+ if const_expr(self.transpose_s):+ with cute.arch.elect_one():+ tcgen05.commit(mbar_ptr + self.mbar_P_consumed_offset + stage)++ def pv_gemm(+ self,+ gemm,+ tOrVi: cute.Tensor,+ sV_cur: cute.Tensor,+ zero_init,+ mbar_ptr: cute.Pointer,+ stage: int,+ phase: Int32,+ ):+ """Issue one PV GEMM, splitting the wait on P only where P is in TMEM.++ With P in tensor memory the instruction can start on its first half and+ wait for the rest mid-flight. A shared-memory A operand has no such+ entry point, so the whole tile must already be published; the caller's+ wait on ``mbar_P_full_O_rescaled`` covers it and the second-half barrier+ is unused.+ """+ if const_expr(self.transpose_s):+ gemm(tCrB=tOrVi, sB=sV_cur, zero_init=zero_init)+ else:+ gemm(+ tCrB=tOrVi,+ sB=sV_cur,+ zero_init=zero_init,+ mbar_ptr=mbar_ptr + self.mbar_P_full_2_offset + stage,+ mbar_phase=phase,+ )++ @cute.jit+ def mma(+ self,+ tiled_mma_qk: cute.ThrMma,+ tiled_mma_pv: cute.ThrMma,+ sQ_terms: Tuple[cute.Tensor, ...],+ sK: cute.Tensor,+ sV: cute.Tensor,+ tStSs: Tuple[cute.Tensor, cute.Tensor],+ tOtOs: tuple[cute.Tensor],+ tOrPs: Tuple[cute.Tensor, cute.Tensor],+ pipeline_kv: cutlass.pipeline.PipelineAsync,+ mbar_ptr: cute.Pointer,+ block_info: BlockInfo,+ num_splits: Int32,+ SeqlenInfoCls: Callable,+ TileSchedulerCls: Callable,+ blocksparse_tensors: Optional[BlockSparseTensors],+ # In smem+ sSFQ_terms: Tuple[Optional[cute.Tensor], ...],+ sSFK: Optional[cute.Tensor],+ sSFV: Optional[cute.Tensor],+ sSFP: Optional[cute.Tensor],+ # In tmem - per-stage scale factors+ tCtSFQ_terms: Tuple[Tuple[cute.Tensor, ...], ...],+ tCtSFKs: Tuple[cute.Tensor, ...],+ tCtSFPs: Tuple[cute.Tensor, ...],+ tCtSFVs: Tuple[cute.Tensor, ...],+ tiled_mma_qk_bf16: Optional[cute.TiledMma] = None,+ tiled_mma_pv_bf16: Optional[cute.TiledMma] = None,+ sK_bf16: Optional[cute.Tensor] = None,+ sV_bf16: Optional[cute.Tensor] = None,+ sQ_bf16: Optional[cute.Tensor] = None,+ mResidualSeqUsedK: Optional[cute.Tensor] = None,+ tOrPs_bf16: Tuple[Optional[cute.Tensor], Optional[cute.Tensor]] = (None, None),+ sP: Optional[cute.Tensor] = None,+ sP_bf16: Optional[cute.Tensor] = None,+ role: cutlass.Constexpr = SplitRole.ALL,+ ):+ runs_pages = const_expr(role != SplitRole.BF16)+ runs_tail = const_expr(+ self.fused_residual_first_block and role != SplitRole.PAGES+ )+ runs_sink = const_expr(self.fused_sink_block and role != SplitRole.PAGES)+ if const_expr(self.fused_residual_first_block):+ tOrV_bf16 = tiled_mma_pv_bf16.make_fragment_B(sV_bf16)+ if const_expr(self.transpose_s):+ tSrK_bf16 = tiled_mma_qk_bf16.make_fragment_A(sK_bf16)+ tSrQ_bf16 = tiled_mma_qk_bf16.make_fragment_B(sQ_bf16)+ else:+ tSrK_bf16 = tiled_mma_qk_bf16.make_fragment_B(sK_bf16)+ tSrQ_bf16 = tiled_mma_qk_bf16.make_fragment_A(sQ_bf16)+ tSrQ_bf16_stages = (tSrQ_bf16[None, None, None, 0], tSrQ_bf16[None, None, None, 0])+ else:+ tSrK_bf16 = None+ tOrV_bf16 = None+ tSrQ_bf16 = None+ tSrQ_bf16_stages = (None, None)+ if const_expr(self.transpose_s):+ tSrQ_terms = tuple(+ tiled_mma_qk.make_fragment_B(sQ_term)+ for sQ_term in sQ_terms+ )+ tSrK = tiled_mma_qk.make_fragment_A(sK)+ else:+ tSrQ_terms = tuple(+ tiled_mma_qk.make_fragment_A(sQ_term)+ for sQ_term in sQ_terms+ )+ tSrK = tiled_mma_qk.make_fragment_B(sK)+ tOrV = tiled_mma_pv.make_fragment_B(sV)+ tSrQ_term_stages = tuple(+ (+ (+ tSrQ[None, None, None, 0],+ tSrQ[None, None, None, 1],+ )+ if const_expr(self.q_stage == 2)+ else (+ tSrQ[None, None, None, 0],+ tSrQ[None, None, None, 0],+ )+ )+ for tSrQ in tSrQ_terms+ )++ qk_mma_op, pv_mma_op = tiled_mma_qk.op, tiled_mma_pv.op+ if const_expr(self.stacked_query_terms):+ # Retain the full-width operand layouts for TMA and BF16 paths.+ # This op consumes their populated prefix and writes exactly the+ # two score ranges that the FP4 softmax joins.+ assert qk_mma_op.shape_mnk == (128, 128, 64)+ narrow_shape = (128, 2 * self.transposed_query_rows, 64)+ if const_expr(self.is_mxfp4):+ qk_mma_op = MmaMXF4Op(+ narrow_shape, tcgen05.CtaGroup.ONE, tcgen05.OperandSource.SMEM+ )+ else:+ qk_mma_op = MmaMXF4NVF4Op(+ self.sf_dtype,+ narrow_shape,+ tcgen05.CtaGroup.ONE,+ tcgen05.OperandSource.SMEM,+ )+ # Q is bound here because it is fixed for the whole tile; K arrives per+ # KV stage and is supplied at the call site through ``qk_gemm``. Which+ # operand each one is depends on whether S is transposed, so bind Q by+ # keyword and let the caller name the K side the same way either way.+ def _qk_scales(term, stage):+ if const_expr(self.transpose_s):+ return dict(+ tScaleA=tCtSFKs[stage],+ tScaleB=tCtSFQ_terms[term][stage],+ )+ return dict(+ tScaleA=tCtSFQ_terms[term][stage],+ tScaleB=tCtSFKs[stage],+ )++ def _make_gemm_si(term, stage):+ common = dict(zero_init=(term == 0))+ if const_expr(self.transpose_s):+ common.update(+ tCrB=tSrQ_term_stages[term][stage],+ sB=sQ_terms[term][None, None, None, stage],+ )+ positional = ()+ else:+ positional = (tSrQ_term_stages[term][stage],)+ common.update(+ sA=sQ_terms[term][None, None, None, stage]+ )+ if const_expr(self.quant_qk):+ return partial(+ sm100_utils.gemm_ptx_partial_fp4,+ qk_mma_op,+ self.tmem_s_offset[stage],+ *positional,+ **common,+ **_qk_scales(term, stage),+ )+ return partial(+ sm100_utils.gemm_ptx_partial,+ qk_mma_op,+ self.tmem_s_offset[stage],+ *positional,+ **common,+ )++ gemm_Si_terms = [+ [+ _make_gemm_si(term, stage)+ for stage in range(self.q_stage)+ ]+ for term in range(self.query_terms)+ ]++ def _make_gemm_pi(stage):+ acc_offset = self.tmem_o_offset[stage if self.q_stage == 2 else 0]+ if const_expr(self.narrow_pv):+ # O^T = V^T P^T. Both operands keep their shared-memory tiles+ # and K-major layouts; the instruction reads the first pv_rows+ # rows of P, the only ones softmax fills.+ narrow_shape = (self.m_block_size, self.pv_rows, 64)+ if const_expr(self.is_mxfp4):+ narrow_pv_op = MmaMXF4Op(+ narrow_shape, tcgen05.CtaGroup.ONE, tcgen05.OperandSource.SMEM+ )+ else:+ narrow_pv_op = MmaMXF4NVF4Op(+ self.sf_dtype,+ narrow_shape,+ tcgen05.CtaGroup.ONE,+ tcgen05.OperandSource.SMEM,+ )++ def gemm_pv_narrow(tCrB, sB, zero_init):+ sm100_utils.gemm_ptx_partial_fp4(+ narrow_pv_op,+ acc_offset,+ tCrB,+ tOrPs[stage],+ sA=sB,+ sB=sP[None, None, None, 0],+ tScaleA=tCtSFVs[stage],+ tScaleB=tCtSFPs[stage],+ zero_init=zero_init,+ )++ return gemm_pv_narrow+ if const_expr(self.transpose_s):+ # A from shared memory: no TMEM address, and the split wait the+ # TMEM path uses is not available here.+ return partial(+ sm100_utils.gemm_ptx_partial_fp4,+ pv_mma_op,+ acc_offset,+ tOrPs[stage],+ sA=sP[None, None, None, 0],+ tScaleA=tCtSFPs[stage],+ tScaleB=tCtSFVs[stage],+ )+ if const_expr(self.quant_pv):+ return partial(+ sm100_utils.gemm_ptx_partial_fp4,+ pv_mma_op,+ acc_offset,+ tOrPs[stage],+ sA=None,+ tScaleA=tCtSFPs[stage],+ tScaleB=tCtSFVs[stage],+ pre_mbar_tiles=self.mbar_p_split(cute.size(tOrPs[stage].shape[2])),+ tA_addr=self.tmem_p_offset[stage],+ )+ return partial(+ sm100_utils.gemm_ptx_partial,+ pv_mma_op,+ acc_offset,+ tOrPs[stage],+ sA=None,+ pre_mbar_tiles=self.mbar_p_split(cute.size(tOrPs[stage].shape[2])),+ )++ gemm_Pi = [_make_gemm_pi(stage) for stage in range(self.q_stage)]+ if const_expr(self.fused_residual_first_block):+ qk_mma_op_bf16 = tiled_mma_qk_bf16.op+ pv_mma_op_bf16 = tiled_mma_pv_bf16.op+ def _make_gemm_si_bf16(stage):+ # Q is bound here and K supplied at the call site, mirroring the+ # FP4 builders; transposed, Q is the B operand.+ if const_expr(self.transpose_s):+ return partial(+ sm100_utils.gemm_ptx_partial,+ qk_mma_op_bf16,+ self.tmem_s_offset[stage],+ tCrB=tSrQ_bf16_stages[stage],+ sB=sQ_bf16[None, None, None, 0],+ zero_init=True,+ )+ return partial(+ sm100_utils.gemm_ptx_partial,+ qk_mma_op_bf16,+ self.tmem_s_offset[stage],+ tSrQ_bf16_stages[stage],+ sA=sQ_bf16[None, None, None, 0],+ zero_init=True,+ )++ gemm_Si_bf16 = [_make_gemm_si_bf16(stage) for stage in range(self.q_stage)]+ gemm_S_sink_bf16 = (+ _make_gemm_si_bf16(1)+ if const_expr(self.fused_sink_block)+ else None+ )+ def _make_gemm_pi_bf16(stage, operand=None):+ tOrP_bf16 = tOrPs_bf16[stage] if operand is None else operand+ if const_expr(self.narrow_pv):+ # The FP4 blocks accumulate O^T, so the residual's PV does+ # too: V^T is an MN-major A operand over the head dimension.+ narrow_pv_op_bf16 = tcgen05.MmaF16BF16Op(+ pv_mma_op_bf16.a_dtype,+ pv_mma_op_bf16.acc_dtype,+ (self.m_block_size, self.pv_rows, 16),+ tcgen05.CtaGroup.ONE,+ tcgen05.OperandSource.SMEM,+ pv_mma_op_bf16.b_major_mode,+ pv_mma_op_bf16.a_major_mode,+ )+ acc_offset = self.tmem_o_offset[stage if self.q_stage == 2 else 0]+ k_blocks = cute.size(tOrP_bf16.shape[2])++ def gemm_pv_bf16_narrow(tCrB, sB, zero_init):+ # The instruction count follows the A operand, so V is+ # cut to as many K blocks as this P carries: one for+ # the sink.+ layout = tCrB.layout+ tCrA = tCrB+ if const_expr(k_blocks != cute.size(layout.shape[2])):+ assert k_blocks == 1+ tCrA = cute.make_tensor(+ tCrB.iterator,+ cute.make_layout(+ (layout.shape[0], layout.shape[1], 1),+ stride=(layout.stride[0], layout.stride[1], 0),+ ),+ )+ sm100_utils.gemm_ptx_partial(+ narrow_pv_op_bf16,+ acc_offset,+ tCrA,+ tOrP_bf16,+ sA=sB,+ sB=sP_bf16[None, None, None, 0],+ zero_init=zero_init,+ )++ return gemm_pv_bf16_narrow+ if const_expr(self.transpose_s):+ # A from shared memory: no split wait, as on the FP4 side.+ return partial(+ sm100_utils.gemm_ptx_partial,+ pv_mma_op_bf16,+ self.tmem_o_offset[stage if self.q_stage == 2 else 0],+ tOrP_bf16,+ sA=sP_bf16[None, None, None, 0],+ )+ return partial(+ sm100_utils.gemm_ptx_partial,+ pv_mma_op_bf16,+ self.tmem_o_offset[stage if self.q_stage == 2 else 0],+ tOrP_bf16,+ sA=None,+ pre_mbar_tiles=self.mbar_p_split(cute.size(tOrP_bf16.shape[2])),+ )++ gemm_Pi_bf16 = [_make_gemm_pi_bf16(stage) for stage in range(self.q_stage)]+ gemm_P_sink_bf16 = (+ _make_gemm_pi_bf16(0, tOrPs_bf16[1])+ if const_expr(self.fused_sink_block)+ else None+ )+ else:+ gemm_Si_bf16 = None+ gemm_S_sink_bf16 = None+ gemm_Pi_bf16 = None+ gemm_P_sink_bf16 = None++ mma_q_consumer_phase = Int32(0)+ mma_kv_consumer_state = cutlass.pipeline.make_pipeline_state(+ cutlass.pipeline.PipelineUserType.Consumer, self.kv_stage+ )+ P_full_O_rescaled_phase = Int32(0)+ residual_kv_full_phase = Int32(0)+ residual_k_full_phase = Int32(0)+ residual_v_full_phase = Int32(0)+ residual_q_full_phase = Int32(0)+ bf16_p_full_phase = Int32(0)+ residual_kv_empty_phase = Int32(1)++ # Partition for S2T copy of SFQ/SFK - change addr per q_stage to avoid overwriting+ if const_expr(self.quant_qk):+ tiled_copy_s2t_sfq_terms_staged = [+ [+ self.mainloop_s2t_copy_and_partition(+ sSFQ_terms[term],+ tCtSFQ_terms[term][stage],+ )+ for stage in range(self.q_stage)+ ]+ for term in range(self.query_terms)+ ]+ tiled_copy_s2t_sfk_staged = [+ self.mainloop_s2t_copy_and_partition(sSFK, tCtSFKs[stage])+ for stage in range(self.q_stage)+ ]+ (+ tiled_copy_s2t_sfq_0,+ tCsSFQ_compact_s2t_0,+ _,+ ) = tiled_copy_s2t_sfq_terms_staged[0][0]+ if const_expr(self.query_terms == 2 and not self.stacked_query_terms):+ (+ tiled_copy_s2t_sfq_1,+ tCsSFQ_compact_s2t_1,+ _,+ ) = tiled_copy_s2t_sfq_terms_staged[1][0]+ else:+ tiled_copy_s2t_sfq_1 = tiled_copy_s2t_sfq_0+ tCsSFQ_compact_s2t_1 = tCsSFQ_compact_s2t_0+ tiled_copy_s2t_sfk, tCsSFK_compact_s2t, _ = tiled_copy_s2t_sfk_staged[0]+ else:+ # Dummy values when quant_qk is False - these won't be used+ tiled_copy_s2t_sfq_terms_staged = []+ tiled_copy_s2t_sfk_staged = []+ tiled_copy_s2t_sfq_0 = None+ tCsSFQ_compact_s2t_0 = None+ tiled_copy_s2t_sfq_1 = None+ tCsSFQ_compact_s2t_1 = None+ tiled_copy_s2t_sfk = None+ tCsSFK_compact_s2t = None++ if const_expr(self.quant_pv):+ tiled_copy_s2t_sfv_staged = [+ self.mainloop_s2t_copy_and_partition(sSFV, tCtSFVs[stage])+ for stage in range(self.q_stage)+ ]+ tiled_copy_s2t_sfv, tCsSFV_compact_s2t, _ = tiled_copy_s2t_sfv_staged[0]+ # S2T copy setup for SFP (P scale factors, computed by softmax warp via R2S)+ tiled_copy_s2t_sfp_staged = [+ self.mainloop_s2t_copy_and_partition(sSFP, tCtSFPs[stage])+ for stage in range(self.q_stage)+ ]+ tiled_copy_s2t_sfp, tCsSFP_compact_s2t, _ = tiled_copy_s2t_sfp_staged[0]+ else:+ tiled_copy_s2t_sfv_staged = []+ tiled_copy_s2t_sfv = None+ tCsSFV_compact_s2t = None+ tiled_copy_s2t_sfp_staged = []+ tiled_copy_s2t_sfp = None+ tCsSFP_compact_s2t = None++ tile_scheduler = TileSchedulerCls()+ work_tile = tile_scheduler.initial_work_tile_info()+++ mma_sfqk_producer_phase = Int32(0)+ while work_tile.is_valid_tile:+ m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx+ seqlen = SeqlenInfoCls(batch_idx)++ block_iter_count = Int32(0)+ process_tile = False++ if const_expr(self.use_block_sparsity):+ block_iter_count = get_total_block_count(blocksparse_tensors, batch_idx, head_idx, m_block)+ process_tile = block_iter_count > Int32(0)+ else:+ n_block_min, n_block_max = self.split_block_range(+ block_info, seqlen, m_block, split_idx, num_splits+ )+ block_iter_count = n_block_max - n_block_min+ process_tile = self.tile_has_work(+ n_block_min, n_block_max, split_idx, num_splits+ )++ if process_tile:+ if const_expr(runs_tail):+ if const_expr(self.fused_sink_block):+ # The rolling residual seeds O before the FP4 pages.+ # The permanent sink is deliberately deferred until the+ # page loop has accumulated O at page-local FP4-P scale.+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_residual_kv_full_offset + 0,+ residual_k_full_phase,+ )+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_residual_kv_full_offset + 2,+ residual_q_full_phase,+ )+ self.qk_gemm(+ gemm_Si_bf16[0],+ tSrK_bf16[None, None, None, 0],+ sK_bf16[None, None, None, 0],+ )+ with cute.arch.elect_one():+ tcgen05.commit(+ mbar_ptr + self.mbar_bf16_S_full_offset+ )+ cute.arch.mbarrier_arrive(+ mbar_ptr+ + self.mbar_residual_kv_empty_offset+ + 0+ )+ residual_k_full_phase ^= 1+ residual_q_full_phase ^= 1++ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_residual_kv_full_offset + 1,+ residual_v_full_phase,+ )+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_bf16_P_full_offset,+ bf16_p_full_phase,+ )+ gemm_Pi_bf16[0](+ tCrB=tOrV_bf16[None, None, None, 0],+ sB=sV_bf16[None, None, None, 0],+ zero_init=True,+ )+ # PV consumes the aliased P/V tiles asynchronously; let+ # the Tensor Core completion release the producer slots.+ with cute.arch.elect_one():+ tcgen05.commit(+ mbar_ptr+ + self.mbar_residual_kv_empty_offset+ + 1+ )+ tcgen05.commit(+ mbar_ptr+ + self.mbar_residual_kv_empty_offset+ + 2+ )+ residual_v_full_phase ^= 1+ bf16_p_full_phase ^= 1+ else:+ # Wait for K and Q (V is consumed later in the PV phase, but+ # the QK GEMM only needs K + Q here).+ iket.range_push("mma_res_wait_k")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_residual_kv_full_offset + 0,+ residual_kv_full_phase,+ )+ iket.range_pop()+ iket.range_push("mma_res_wait_q")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_residual_kv_full_offset + 2,+ residual_kv_full_phase,+ )+ iket.range_pop()+ self.qk_gemm(+ gemm_Si_bf16[0],+ tSrK_bf16[None, None, None, 0],+ sK_bf16[None, None, None, 0],+ )+ with cute.arch.elect_one():+ tcgen05.commit(mbar_ptr + self.mbar_bf16_S_full_offset)+ # Wait for V (consumed in BF16 GEMM_PV below).+ iket.range_push("mma_res_wait_v")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_residual_kv_full_offset + 1,+ residual_kv_full_phase,+ )+ iket.range_pop()+ iket.range_push("mma_res_wait_p")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_bf16_P_full_offset,+ Int32(0),+ )+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_bf16_P_full_2_offset,+ Int32(0),+ )+ iket.range_pop()+ # BF16 PV: P_BF16 (TMEM) × V_BF16 (SMEM) → O TMEM (zero_init=True).+ gemm_Pi_bf16[0](+ tCrB=tOrV_bf16[None, None, None, 0],+ sB=sV_bf16[None, None, None, 0],+ zero_init=True,+ )+ with cute.arch.elect_one():+ cute.arch.mbarrier_arrive(+ mbar_ptr + self.mbar_residual_kv_empty_offset + 0,+ )+ cute.arch.mbarrier_arrive(+ mbar_ptr + self.mbar_residual_kv_empty_offset + 1,+ )+ cute.arch.mbarrier_arrive(+ mbar_ptr + self.mbar_residual_kv_empty_offset + 2,+ )+ residual_kv_full_phase ^= 1+ if const_expr(runs_pages):+ for stage in cutlass.range_constexpr(self.q_stage):+ iket.range_push("mma_wait_qk")+ if const_expr(self.q_replicate > 1 or self.stacked_query_terms):+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_q_fp4_ready_offset + stage,+ mma_q_consumer_phase,+ )+ else:+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_load_q_full_offset + stage,+ mma_q_consumer_phase,+ )+ # 2. wait for K0+ if const_expr(stage == 0):+ pipeline_kv.consumer_wait(mma_kv_consumer_state)+ iket.range_pop()+ tSrKi = tSrK[None, None, None, mma_kv_consumer_state.index]+ # We don't need to acquire empty S0 / S1.+ # For the first iteration, we don't need to wait as we're guaranteed S0 / S1+ # are empty. For subsequent iterations, the wait happened at the end+ # of the while loop.++ # Copy SFQ+ # only tmem changes per q_stage.+ if const_expr(self.quant_qk):+ sm100_utils.tcgen05_after_thread_sync()+ iket.range_push("mma_wait_sf")+ cute.arch.mbarrier_wait(mbar_ptr + self.mbar_sfqk_load_offset + stage, mma_sfqk_producer_phase)+ iket.range_pop()+ _, _, tCtSFQ_compact_s2t_0 = (+ tiled_copy_s2t_sfq_terms_staged[0][stage]+ )+ cute.copy(+ tiled_copy_s2t_sfq_0,+ tCsSFQ_compact_s2t_0[+ None, None, None, None, stage+ ],+ tCtSFQ_compact_s2t_0,+ )+ if const_expr(self.query_terms == 2 and not self.stacked_query_terms):+ _, _, tCtSFQ_compact_s2t_1 = (+ tiled_copy_s2t_sfq_terms_staged[1][stage]+ )+ cute.copy(+ tiled_copy_s2t_sfq_1,+ tCsSFQ_compact_s2t_1[+ None, None, None, None, stage+ ],+ tCtSFQ_compact_s2t_1,+ )++ _, _, tCtSFK_compact_s2t = tiled_copy_s2t_sfk_staged[stage]+ tCsSFK_compact_s2t_staged = tCsSFK_compact_s2t[None, None, None, None, mma_kv_consumer_state.index]+ cute.copy(+ tiled_copy_s2t_sfk,+ tCsSFK_compact_s2t_staged,+ tCtSFK_compact_s2t,+ )++ # 3. gemm+ # tiled_mma_qk = sm100_utils.gemm(tiled_mma_qk, tStSs[stage], tSrQs[stage], tSrKi, zero_init=True)+ sK_cur = sK[None, None, None, mma_kv_consumer_state.index]+ if const_expr(self.uneven_kv_smem):+ sK_cur = self.offset_kv_smem(+ sK_cur, mma_kv_consumer_state.index, mma_kv_consumer_state.phase+ )++ self.qk_gemm(+ gemm_Si_terms[0][stage],+ tSrKi,+ sK_cur,+ )+ if const_expr(self.query_terms == 2 and not self.stacked_query_terms):+ self.qk_gemm(+ gemm_Si_terms[1][stage],+ tSrKi,+ sK_cur,+ )++ # 4. release S0 / S1+ with cute.arch.elect_one():+ tcgen05.commit(mbar_ptr + self.mbar_S_full_offset + stage)+ mma_q_consumer_phase ^= 1+ if const_expr(self.quant_qk):+ mma_sfqk_producer_phase ^= 1+ # 5. release K0+ pipeline_kv.consumer_release(mma_kv_consumer_state)+ mma_kv_consumer_state.advance()+ # End of GEMM (Q1 * K0 -> S1)+ # Note: Q0 & Q1 are still needed in the seqlen_kv loop+ # so we need to release them after the seqlen_kv loop++ block_loop_count = block_iter_count - 1+ # A tail ahead of the pages has already written O.+ O_should_accumulate = runs_tail+ for i in cutlass.range(block_loop_count, unroll=1):+ # GEMM_PV00 (P0 * V0 -> O0_partial), O0 needs to be accumulated in the seqlen_kv loop+ # 1. wait for V0+ iket.range_push("mma_wait_v")+ pipeline_kv.consumer_wait(mma_kv_consumer_state)+ iket.range_pop()+ mma_kv_release_state = mma_kv_consumer_state.clone()+ Vi_index, Vi_phase = mma_kv_consumer_state.index, mma_kv_consumer_state.phase+ tOrVi = tOrV[None, None, None, Vi_index]+ for stage in cutlass.range_constexpr(self.q_stage):+ def issue_pv():+ # 2. acquire corrected O0/O1_partial and P0 / P1+ # For the first iteration in this work tile, waiting for O0/O1_partial+ # means that the correction warps has finished reading tO during+ # the last iteration of the previous work tile has finished.+ iket.range_push("mma_wait_p")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage,+ P_full_O_rescaled_phase,+ )+ iket.range_pop()+ # No need for mbar.wait because it depends on the same Si as the prev qk mma+ if const_expr(self.quant_pv):+ # S2T copy SFP: sSFP (smem) -> tCtSFPs (tmem)+ _, _, tCtSFP_compact_s2t = tiled_copy_s2t_sfp_staged[stage]+ tCsSFP_compact_s2t_cur = tCsSFP_compact_s2t[None, None, None, None, stage]+ cute.copy(+ tiled_copy_s2t_sfp,+ tCsSFP_compact_s2t_cur,+ tCtSFP_compact_s2t,+ )+ # S2T copy SFV: sSFV (smem) -> tCtSFVs (tmem)+ _, _, tCtSFV_compact_s2t = tiled_copy_s2t_sfv_staged[stage]+ tCsSFV_compact_s2t_staged = tCsSFV_compact_s2t[None, None, None, None, Vi_index]+ cute.copy(+ tiled_copy_s2t_sfv,+ tCsSFV_compact_s2t_staged,+ tCtSFV_compact_s2t,+ )++ sV_cur = sV[None, None, None, Vi_index]+ if const_expr(self.uneven_kv_smem):+ sV_cur = self.offset_kv_smem(sV_cur, Vi_index, Vi_phase)++ self.pv_gemm(+ gemm_Pi[stage],+ tOrVi,+ sV_cur,+ self.p_scale_two_level or not O_should_accumulate,+ mbar_ptr,+ stage,+ P_full_O_rescaled_phase,+ )+ self.release_shared_p(mbar_ptr, stage)++ if const_expr(stage == self.q_stage - 1):+ pipeline_kv.consumer_release(mma_kv_release_state)+ mma_kv_release_state.advance()+ # End of GEMM_PV00 (P0 * V0 -> O0_partial)++ def issue_qk():+ # GEMM_QK0i (Q0 * Ki -> S0)+ # 1. wait for Ki+ if const_expr(stage == 0):+ mma_kv_consumer_state.advance()+ iket.range_push("mma_wait_k")+ pipeline_kv.consumer_wait(mma_kv_consumer_state)+ iket.range_pop()+ Ki_index, Ki_phase = mma_kv_consumer_state.index, mma_kv_consumer_state.phase+ # The wait below is what makes overwriting Si safe: the+ # softmax arrives on it once its tensor-memory load has+ # retired, which also implies the previous QK finished.+ if const_expr(self.quant_qk):+ sm100_utils.tcgen05_after_thread_sync()+ iket.range_push("mma_wait_sf")+ cute.arch.mbarrier_wait(mbar_ptr + self.mbar_sfqk_load_offset + stage, mma_sfqk_producer_phase)+ iket.range_pop()+ # One Q stage keeps its scales in the unused+ # S stage throughout the page loop. The first+ # page loads them again for every new tile;+ # the sink reuses those columns only after the loop.+ # Two Q stages overwrite each other's scales.+ if const_expr(self.q_stage == 2):+ _, _, tCtSFQ_compact_s2t_0 = (+ tiled_copy_s2t_sfq_terms_staged[0][stage]+ )+ cute.copy(+ tiled_copy_s2t_sfq_0,+ tCsSFQ_compact_s2t_0[+ None, None, None, None, stage+ ],+ tCtSFQ_compact_s2t_0,+ )+ if const_expr(self.query_terms == 2 and not self.stacked_query_terms):+ _, _, tCtSFQ_compact_s2t_1 = (+ tiled_copy_s2t_sfq_terms_staged[1][stage]+ )+ cute.copy(+ tiled_copy_s2t_sfq_1,+ tCsSFQ_compact_s2t_1[+ None, None, None, None, stage+ ],+ tCtSFQ_compact_s2t_1,+ )+ _, _, tCtSFK_compact_s2t = tiled_copy_s2t_sfk_staged[stage]+ tCsSFK_compact_s2t_staged = tCsSFK_compact_s2t[None, None, None, None, mma_kv_consumer_state.index]+ cute.copy(+ tiled_copy_s2t_sfk,+ tCsSFK_compact_s2t_staged,+ tCtSFK_compact_s2t,+ )++ sK_cur = sK[None, None, None, Ki_index]+ if const_expr(self.uneven_kv_smem):+ sK_cur = self.offset_kv_smem(sK_cur, Ki_index, Ki_phase)++ self.qk_gemm(+ gemm_Si_terms[0][stage],+ tSrK[None, None, None, Ki_index],+ sK_cur,+ )+ if const_expr(self.query_terms == 2 and not self.stacked_query_terms):+ self.qk_gemm(+ gemm_Si_terms[1][stage],+ tSrK[None, None, None, Ki_index],+ sK_cur,+ )+ # Release Si. The commit fires once every MMA this+ # warp issued before it has retired, so it belongs+ # next to the QK: leaving it after the PV would make+ # the softmax wait for a GEMM it does not read.+ with cute.arch.elect_one():+ tcgen05.commit(mbar_ptr + self.mbar_S_full_offset + stage)++ if const_expr(self.tmem_sfpv_offset is not None):+ # Nothing the next QK overwrites is still live. P is in+ # shared memory here and its scale factors have tensor+ # memory of their own, so S is the only thing left in+ # these columns and the softmax released it at the top+ # of its step. Issuing the QK before waiting on P is+ # what lets it run underneath the softmax instead of+ # after it, which is the whole of the block's exposed+ # GEMM latency.+ issue_qk()+ issue_pv()+ else:+ # P and its scale factors share S's columns, so the+ # next QK cannot be issued until this PV has consumed+ # them.+ issue_pv()+ issue_qk()+ # 4. release Ki+ pipeline_kv.consumer_release(mma_kv_consumer_state)+ mma_kv_consumer_state.advance()+ P_full_O_rescaled_phase ^= 1+ if const_expr(self.quant_qk):+ mma_sfqk_producer_phase ^= 1+ O_should_accumulate = True+ # End of seqlen_kv loop++ # release Q0 & Q1+ with cute.arch.elect_one():+ for stage in cutlass.range_constexpr(self.q_stage):+ tcgen05.commit(mbar_ptr + self.mbar_load_q_empty_offset + stage)++ # GEMM_PV00 (P0 * V0 -> O0_partial), O0 needs to be accumulated in the seqlen_kv loop+ # 1. wait for V0+ iket.range_push("mma_wait_v")+ pipeline_kv.consumer_wait(mma_kv_consumer_state)+ iket.range_pop()+ Vi_index, Vi_phase = mma_kv_consumer_state.index, mma_kv_consumer_state.phase+ tOrVi = tOrV[None, None, None, Vi_index]+ for stage in cutlass.range_constexpr(self.q_stage):+ # 2. acquire corrected Oi_partial and Pi+ iket.range_push("mma_wait_p")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage, P_full_O_rescaled_phase+ )+ iket.range_pop()+ # 3. gemm+ # sm100_utils.gemm(tiled_mma_pv, tOtO0, tOrP0, tOrVi, zero_init=True)+ # gemm_Pi[stage](tCrB=tOrVi, sB=sV[None, None, None, Vi_index], zero_init=not O_should_accumulate)++ if const_expr(self.quant_pv):+ # S2T copy SFP: sSFP (smem) -> tCtSFPs (tmem)+ _, _, tCtSFP_compact_s2t = tiled_copy_s2t_sfp_staged[stage]+ tCsSFP_compact_s2t_cur = tCsSFP_compact_s2t[None, None, None, None, stage]+ cute.copy(+ tiled_copy_s2t_sfp,+ tCsSFP_compact_s2t_cur,+ tCtSFP_compact_s2t,+ )+ # S2T copy SFV: sSFV (smem) -> tCtSFVs (tmem)+ _, _, tCtSFV_compact_s2t = tiled_copy_s2t_sfv_staged[stage]+ tCsSFV_compact_s2t_staged = tCsSFV_compact_s2t[None, None, None, None, Vi_index]+ cute.copy(+ tiled_copy_s2t_sfv,+ tCsSFV_compact_s2t_staged,+ tCtSFV_compact_s2t,+ )+ sV_cur = sV[None, None, None, Vi_index]+ if const_expr(self.uneven_kv_smem):+ sV_cur = self.offset_kv_smem(sV_cur, Vi_index, Vi_phase)+ _zi_post = self.p_scale_two_level or not O_should_accumulate+ self.pv_gemm(+ gemm_Pi[stage],+ tOrVi,+ sV_cur,+ _zi_post,+ mbar_ptr,+ stage,+ P_full_O_rescaled_phase,+ )+ self.release_shared_p(mbar_ptr, stage)+ # 4. release accumulated O0_partial+ # We do need O_full here since for the last tile, by the time the softmax warp+ # has signaled to the correction warps, the softmax warp has just finished compute+ # the row sum of the current tile. It does not guarantee that the 1st tile+ # of the next work tile has been computed yet.+ if const_expr(not runs_sink):+ with cute.arch.elect_one():+ tcgen05.commit(mbar_ptr + self.mbar_O_full_offset + stage)+ if const_expr(self.transpose_s and self.is_persistent):+ # No later page observes this tile's final P phase.+ # Close it before another persistent tile can reuse+ # the barrier and overwrite the shared P tile.+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_P_consumed_offset + stage,+ P_full_O_rescaled_phase,+ )+ # End of GEMM_PV00 (P0 * V0 -> O0_partial)+ P_full_O_rescaled_phase ^= 1+ # 5. release Vi_end+ pipeline_kv.consumer_release(mma_kv_consumer_state)+ mma_kv_consumer_state.advance()+ # End of GEMM_PV1(i_end) (P1 * Vi_end -> O1)+ elif const_expr(not runs_sink):+ # The tail's PV is the last GEMM of a split with no page+ # and no sink.+ with cute.arch.elect_one():+ tcgen05.commit(mbar_ptr + self.mbar_O_full_offset)++ if const_expr(runs_sink):+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_residual_kv_full_offset + 0,+ residual_k_full_phase,+ )+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_residual_kv_full_offset + 2,+ residual_q_full_phase,+ )+ self.qk_gemm(+ gemm_S_sink_bf16,+ tSrK_bf16[None, None, None, 0],+ sK_bf16[None, None, None, 0],+ )+ with cute.arch.elect_one():+ tcgen05.commit(mbar_ptr + self.mbar_S_full_offset)+ cute.arch.mbarrier_arrive(+ mbar_ptr+ + self.mbar_residual_kv_empty_offset+ + 0+ )+ residual_k_full_phase ^= 1+ residual_q_full_phase ^= 1++ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_residual_kv_full_offset + 1,+ residual_v_full_phase,+ )+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_bf16_P_full_offset,+ bf16_p_full_phase,+ )+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_P_full_O_rescaled_offset,+ P_full_O_rescaled_phase,+ )+ gemm_P_sink_bf16(+ tCrB=tOrV_bf16[None, None, None, 0],+ sB=sV_bf16[None, None, None, 0],+ zero_init=self.p_scale_two_level and runs_pages,+ )+ with cute.arch.elect_one():+ tcgen05.commit(+ mbar_ptr + self.mbar_O_full_offset+ )+ cute.arch.mbarrier_arrive(+ mbar_ptr+ + self.mbar_residual_kv_empty_offset+ + 1+ )+ cute.arch.mbarrier_arrive(+ mbar_ptr+ + self.mbar_residual_kv_empty_offset+ + 2+ )+ residual_v_full_phase ^= 1+ bf16_p_full_phase ^= 1+ P_full_O_rescaled_phase ^= 1+++ # Advance to next tile+ tile_scheduler.advance_to_next_work()+ work_tile = tile_scheduler.get_current_work()+ # End of persistent scheduler loop+++ @cute.jit+ def softmax_warpgroup_reduce(+ self,+ vals: cute.Tensor,+ sSoftmaxRed: cute.Tensor,+ tidx: Int32,+ red_buf: Int32,+ op: Callable,+ intra_warp: cutlass.Constexpr[bool] = True,+ row_group: cutlass.Constexpr[int] = 0,+ ) -> Int32:+ """Reduce one value per query row across all softmax threads, in place.++ Transposed, a row of S runs along the thread axis, so a row reduction is+ a warp butterfly followed by an exchange between the four softmax warps.+ The scratch is double buffered by ``red_buf`` so a single named barrier+ per call is enough: a warp can only overwrite the buffer it read two+ calls ago, and the intervening barrier already ordered that read.++ ``intra_warp`` is for callers that already hold the warp-wide result+ because they needed one of the butterfly's intermediate stages anyway.+ """+ num_warps = const_expr(len(self.softmax0_warp_ids))+ rows = const_expr(self.softmax_rows_per_group)+ if const_expr(intra_warp):+ for j in cutlass.range_constexpr(rows):+ vals[j] = utils.warp_reduce(vals[j], op)+ base = const_expr(row_group * 2 * num_warps * rows) + red_buf * (num_warps * rows)+ if tidx % cute.arch.WARP_SIZE == 0:+ warp = tidx // cute.arch.WARP_SIZE+ for j in cutlass.range_constexpr(rows):+ sSoftmaxRed[base + warp * rows + j] = vals[j]+ cute.arch.barrier(+ barrier_id=int(NamedBarrierFwd.SoftmaxReduce) + row_group,+ number_of_threads=num_warps * cute.arch.WARP_SIZE,+ )+ for j in cutlass.range_constexpr(rows):+ acc = sSoftmaxRed[base + j]+ for w in cutlass.range_constexpr(1, num_warps):+ acc = op(acc, sSoftmaxRed[base + w * rows + j])+ vals[j] = acc+ return red_buf ^ 1++ @cute.jit+ def softmax_wait_corr(+ self, mbar_ptr: cute.Pointer, stage: int | Int32, si_corr_producer_phase: Int32+ ) -> Int32:+ """Wait until correction has taken the factors last published in+ sScale, and return the producer phase for the next wait."""+ iket.range_push("sm_wait_corr")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_softmax_corr_empty_offset + stage, si_corr_producer_phase+ )+ iket.range_pop()+ return si_corr_producer_phase ^ 1++ def scatter_lane(self, row: int) -> int:+ """The lane within a cluster of ``softmax_rows_per_group`` that ends a+ reduce-scatter holding ``row``: the row's bits in reverse order, since+ the first butterfly step splits the rows in halves."""+ steps = self.softmax_rows_per_group.bit_length() - 1+ return int(format(row, f"0{steps}b")[::-1], 2) if steps else 0++ @cute.jit+ def scatter_row(self, tidx: Int32) -> Int32:+ """The row a lane holds after the reduce-scatter, ``scatter_lane``+ inverted."""+ rows = const_expr(self.softmax_rows_per_group)+ row = Int32(0)+ for i in cutlass.range_constexpr(rows.bit_length() - 1):+ row += ((tidx >> i) & 1) * (rows >> (i + 1))+ return row++ @cute.jit+ def softmax_warpgroup_reduce_scattered(+ self,+ val: Float32,+ own_row: Int32,+ block_max: cute.Tensor,+ sSoftmaxRed: cute.Tensor,+ tidx: Int32,+ red_buf: Int32,+ row_group: cutlass.Constexpr[int] = 0,+ ) -> Int32:+ """``softmax_warpgroup_reduce`` for a warp maximum reduce-scattered over+ lanes: the first lanes of each warp hold one row each, and every thread+ reads every row back, so ``block_max`` ends as the unscattered reduce+ leaves it. Same scratch, slots and barrier."""+ num_warps = const_expr(len(self.softmax0_warp_ids))+ rows = const_expr(self.softmax_rows_per_group)+ base = const_expr(row_group * 2 * num_warps * rows) + red_buf * (num_warps * rows)+ if tidx % cute.arch.WARP_SIZE < rows:+ warp = tidx // cute.arch.WARP_SIZE+ sSoftmaxRed[base + warp * rows + own_row] = val+ cute.arch.barrier(+ barrier_id=int(NamedBarrierFwd.SoftmaxReduce) + row_group,+ number_of_threads=num_warps * cute.arch.WARP_SIZE,+ )+ for j in cutlass.range_constexpr(rows):+ acc = sSoftmaxRed[base + j]+ for w in cutlass.range_constexpr(1, num_warps):+ acc = cute.arch.fmax(acc, sSoftmaxRed[base + w * rows + j])+ block_max[j] = acc+ return red_buf ^ 1++ @cute.jit+ def split_block_range(+ self,+ block_info: BlockInfo,+ seqlen: SeqlenInfoQK,+ m_block: Int32,+ split_idx: Int32,+ num_splits: Int32,+ ) -> Tuple[Int32, Int32]:+ """The FP4 page blocks a split walks, ``[n_block_min, n_block_max)``.++ With a split of its own for the BF16 blocks, ``num_splits`` counts it+ too: the pages are divided among the others exactly as a launch with+ no BF16 block would divide them, and the last split is handed none.+ Every warp role must ask here, since a role that saw a different range+ would wait on barriers the others never arrive on.+ """+ if const_expr(not self.bf16_split):+ return block_info.get_n_block_min_max(+ seqlen, m_block, split_idx, num_splits+ )+ n_block_min, n_block_max = block_info.get_n_block_min_max(+ seqlen, m_block, split_idx, num_splits - 1+ )+ if split_idx == num_splits - 1:+ n_block_max = n_block_min+ return n_block_min, n_block_max++ @cute.jit+ def is_bf16_split(self, split_idx: Int32, num_splits: Int32):+ """Whether this is the split that merges the tail and the sink."""+ if const_expr(self.bf16_split):+ return split_idx == num_splits - 1+ return split_idx == self.residual_split_idx++ @cute.jit+ def tile_has_work(+ self,+ n_block_min: Int32,+ n_block_max: Int32,+ split_idx: Int32,+ num_splits: Int32,+ ):+ """Whether a work tile has anything for the pipeline to do.++ Without split-k every tile runs, including one whose fp4 length is+ zero: its single mainloop step is fully masked and the epilogue still+ owes an output row. Split-k drops empty splits instead, except for the+ one that owns the residual block, which has work whether or not any fp4+ block landed in it.++ Every warp asks this question separately and they have to agree, so+ they all ask it here.+ """+ if const_expr(not self.is_split_kv):+ return True+ has_work = n_block_min < n_block_max+ if const_expr(self.fused_residual_first_block):+ if self.is_bf16_split(split_idx, num_splits):+ has_work = cutlass.Boolean(True)+ return has_work++ @cute.jit+ def run_split_role(self, is_bf16_cta, role_body: Callable):+ """Run one warp role's body, specialised for what this CTA merges.++ ``role_body`` takes the :class:`SplitRole` as a compile-time keyword,+ so each CTA runs one of two separately compiled bodies when the BF16+ blocks have a split of their own, and the one combined body otherwise.+ A CTA owns a single tile, which is what keeps every barrier phase of+ one body independent of the other's.+ """+ if const_expr(self.bf16_split):+ if is_bf16_cta:+ role_body(role=SplitRole.BF16)+ else:+ role_body(role=SplitRole.PAGES)+ else:+ role_body(role=SplitRole.ALL)++ @cute.jit+ def zero_transposed_p(self, sP_uint8: cute.Tensor, sSFP: cute.Tensor, tidx: Int32):+ """Zero the shared P tile and its scale factors once per CTA.++ Only the first ``transposed_query_rows`` rows are ever written, but the+ MMA reads all 128 and an undefined E4M3 scale factor can decode to NaN.+ Zeroing the whole region is a permutation-invariant write, so the+ swizzle can be ignored and the buffers filled as flat words.+ """+ num_threads = const_expr(len(self.softmax0_warp_ids) * cute.arch.WARP_SIZE)+ p_words = const_expr(self.m_block_size * self.n_block_size // 2 // 4)+ sfp_words = const_expr(+ self.m_block_size * self.head_dim_v_padded // self.sf_vec_size * self.q_stage // 4+ )+ p_flat = cute.make_tensor(+ cute.recast_ptr(sP_uint8.iterator, dtype=Int32), cute.make_layout(p_words)+ )+ sfp_flat = cute.make_tensor(+ cute.recast_ptr(sSFP.iterator, dtype=Int32), cute.make_layout(sfp_words)+ )+ for i in cutlass.range_constexpr(p_words // num_threads):+ p_flat[tidx + i * num_threads] = Int32(0)+ for i in cutlass.range_constexpr(max(1, sfp_words // num_threads)):+ if tidx + i * num_threads < sfp_words:+ sfp_flat[tidx + i * num_threads] = Int32(0)++ @cute.jit+ def zero_transposed_p_bf16(+ self,+ sP_bf16: cute.Tensor,+ sP_bf16_mk: cute.Tensor,+ tidx: Int32,+ row_group: cutlass.Constexpr[int] = 0,+ ):+ """Zero the shared BF16 P tile of the residual block.++ The FP4 tile is cleared once per CTA, but this one shares its bytes+ with the residual Q tile and so cannot be cleared until the QK GEMM+ has consumed it. As above the write is permutation invariant, so the+ swizzle can be ignored and the buffer filled as flat words.++ Several row groups wake on the same S and nothing orders one group's+ clear against another's stores, so then each clears only its share of+ the rows past ``transposed_query_rows``: a group stores every kv+ position of its own rows, and no group stores into those.+ """+ groups = const_expr(self.softmax_row_groups)+ if const_expr(groups == 1):+ num_threads = const_expr(len(self.softmax0_warp_ids) * cute.arch.WARP_SIZE)+ words = const_expr(self.m_block_size * self.n_block_size * 2 // 4)+ flat = cute.make_tensor(+ cute.recast_ptr(sP_bf16.iterator, dtype=Int32), cute.make_layout(words)+ )+ for i in cutlass.range_constexpr(words // num_threads):+ flat[tidx + i * num_threads] = Int32(0)+ else:+ share = const_expr(+ (self.m_block_size - self.transposed_query_rows) // groups+ )+ first = const_expr(self.transposed_query_rows + row_group * share)+ for i in cutlass.range_constexpr(share):+ sP_bf16_mk[first + i, tidx] = cutlass.BFloat16(0.0)++ @cute.jit+ def softmax_residual_step_transposed(+ self,+ softmax_scale_log2: Float32,+ mbar_ptr: cute.Pointer,+ thr_tmem_load: cute.CopyAtom,+ tStS_t2r: cute.Tensor,+ frg_shape,+ row_max: cute.Tensor,+ row_sum: cute.Tensor,+ sP_bf16: cute.Tensor,+ sP_bf16_mk: cute.Tensor,+ sSoftmaxRed: cute.Tensor,+ red_buf: Int32,+ tidx: Int32,+ batch_idx: Int32,+ split_idx: Int32,+ mResidualSeqUsedK: Optional[cute.Tensor],+ mSinkSeqUsedK: Optional[cute.Tensor] = None,+ seqused_fp4: Int32 = Int32(1),+ row_group: cutlass.Constexpr[int] = 0,+ pages_follow: cutlass.Constexpr[bool] = True,+ ) -> Int32:+ """The residual block in the same orientation as the FP4 blocks.++ Structurally ``softmax_step_transposed`` with the quantization replaced+ by a plain bf16 store, since P feeds a bf16 MMA and carries no scale+ factors. It runs before any FP4 block and seeds the online softmax, so+ ``row_max`` and ``row_sum`` leave here holding the residual's+ contribution and the FP4 blocks merge onto them with is_first=False.+ ``pages_follow`` is whether this CTA has FP4 blocks to run after it.+ """+ rows = const_expr(self.softmax_rows_per_group)+ row_base = const_expr(row_group * rows)++ iket.range_push("sm_res_wait_s")+ cute.arch.mbarrier_wait(mbar_ptr + self.mbar_bf16_S_full_offset, Int32(0))+ iket.range_pop()++ iket.range_push("sm_res_comp")+ # Safe here and only here: the QK GEMM has just released the Q tile+ # these bytes belong to. With one row group the row-maximum exchange+ # below carries the barrier that separates this clear from the stores+ # at the end; with more the groups clear disjoint rows that none of+ # them stores into.+ self.zero_transposed_p_bf16(sP_bf16, sP_bf16_mk, tidx, row_group=row_group)++ tSrS = cute.make_rmem_tensor(frg_shape, self.qk_acc_dtype)+ cute.copy(thr_tmem_load, tStS_t2r, tSrS)+ cute.arch.fence_view_async_tmem_load()+ if const_expr(mSinkSeqUsedK is not None and pages_follow):+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_sfqk_load_offset)++ # Transposed, thread ``tidx`` owns kv position ``tidx``, so the residual+ # length is one predicate. A split that does not own the residual sees a+ # length of zero, masks every position, and contributes nothing.+ seqused_k = Int32(self.n_block_size)+ residual_start = Int32(0)+ if const_expr(mResidualSeqUsedK is not None):+ seqused_k = mResidualSeqUsedK[batch_idx]+ if const_expr(self.is_split_kv and not self.bf16_split):+ if split_idx != self.residual_split_idx:+ seqused_k = Int32(0)+ if const_expr(mSinkSeqUsedK is not None):+ sink_length = mSinkSeqUsedK[batch_idx]+ if seqused_fp4 == Int32(0):+ residual_start = sink_length+ if (tidx < residual_start) or (tidx >= seqused_k):+ for j in cutlass.range_constexpr(rows):+ tSrS[j] = -Float32.inf++ block_max = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ for j in cutlass.range_constexpr(rows):+ block_max[j] = utils.warp_reduce(tSrS[j], cute.arch.fmax)+ red_buf = self.softmax_warpgroup_reduce(+ block_max,+ sSoftmaxRed,+ tidx,+ red_buf,+ cute.arch.fmax,+ intra_warp=False,+ row_group=row_group,+ )+ max_scaled = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ for j in cutlass.range_constexpr(rows):+ row_max[j] = block_max[j]+ safe = block_max[j] if block_max[j] != -Float32.inf else 0.0+ max_scaled[j] = safe * softmax_scale_log2+ if const_expr(self.p_scale_rescale_log2):+ max_scaled[j] -= Float32(self.p_scale_rescale_log2)++ for j in cutlass.range_constexpr(rows):+ tSrS[j] = cute.arch.exp2(tSrS[j] * softmax_scale_log2 - max_scaled[j])+ row_sum[j] = tSrS[j]++ tSrP = cute.make_rmem_tensor(frg_shape, cutlass.BFloat16)+ tSrP.store(tSrS.load().to(cutlass.BFloat16))+ for j in cutlass.range_constexpr(rows):+ sP_bf16_mk[row_base + j, tidx] = tSrP[j]+ # The MMA reads P through the async proxy while these are generic-proxy+ # stores, so the mbarrier below needs the fence to make them visible.+ cute.arch.fence_view_async_shared()+ iket.range_pop()++ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_bf16_P_full_offset)+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_bf16_P_full_2_offset)+ return red_buf++ @cute.jit+ def softmax_sink_step_transposed(+ self,+ mma_si_consumer_phase: Int32,+ si_corr_producer_phase: Int32,+ red_buf: Int32,+ softmax_scale_log2: Float32,+ mbar_ptr: cute.Pointer,+ thr_tmem_load: cute.CopyAtom,+ tStS_t2r: cute.Tensor,+ tStS_sink_t2r: cute.Tensor,+ frg_shape,+ row_max: cute.Tensor,+ row_sum: cute.Tensor,+ sScale: cute.Tensor,+ sP_bf16: cute.Tensor,+ sP_bf16_mk: cute.Tensor,+ sSoftmaxRed: cute.Tensor,+ stage: int | Int32,+ tidx: Int32,+ batch_idx: Int32,+ mSinkSeqUsedK: cute.Tensor,+ split_idx: Int32,+ row_group: cutlass.Constexpr[int] = 0,+ waits_for_previous: cutlass.Constexpr[bool] = True,+ ) -> Tuple[Int32, Int32, Int32]:+ """Merge the permanent BF16 sink after every FP4 page.++ ``waits_for_previous`` as for ``softmax_step_transposed``: false only+ when nothing in the tile published before it.++ The sink's probabilities are kept in BF16, so its large score can+ rescale the FP32 output accumulator without forcing the FP4 page P+ tiles to quantize against the sink maximum.+ """+ rows = const_expr(self.softmax_rows_per_group)+ row_base = const_expr(row_group * rows)++ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_S_full_offset + stage,+ mma_si_consumer_phase,+ )+ tSrS = cute.make_rmem_tensor(frg_shape, self.qk_acc_dtype)+ cute.copy(thr_tmem_load, tStS_sink_t2r, tSrS)+ cute.arch.fence_view_async_tmem_load()++ sink_length = mSinkSeqUsedK[batch_idx]+ if const_expr(self.is_split_kv and not self.bf16_split):+ # One sink, one owner, as with the tail: every other split merges+ # a length of zero, so the combine sees the sink exactly once.+ if split_idx != self.residual_split_idx:+ sink_length = Int32(0)+ if tidx >= sink_length:+ for j in cutlass.range_constexpr(rows):+ tSrS[j] = -Float32.inf++ block_max = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ for j in cutlass.range_constexpr(rows):+ block_max[j] = utils.warp_reduce(tSrS[j], cute.arch.fmax)+ red_buf = self.softmax_warpgroup_reduce(+ block_max,+ sSoftmaxRed,+ tidx,+ red_buf,+ cute.arch.fmax,+ intra_warp=False,+ row_group=row_group,+ )++ acc_scale = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ row_max_safe = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ for j in cutlass.range_constexpr(rows):+ row_max_old = row_max[j]+ row_max_new = cute.arch.fmax(row_max_old, block_max[j])+ safe = row_max_new if row_max_new != -Float32.inf else 0.0+ acc_scale_ = (row_max_old - safe) * softmax_scale_log2+ scale = utils.exp2f(acc_scale_)+ if const_expr(RESCALE_THRESHOLD > 0.0):+ if acc_scale_ >= -RESCALE_THRESHOLD:+ row_max_new = row_max_old+ safe = row_max_old+ scale = Float32(1.0)+ row_max[j] = row_max_new+ row_max_safe[j] = safe+ acc_scale[j] = scale++ if const_expr(waits_for_previous):+ si_corr_producer_phase = self.softmax_wait_corr(+ mbar_ptr, stage, si_corr_producer_phase+ )+ if tidx == 0:+ for j in cutlass.range_constexpr(rows):+ sScale[row_base + j + stage * self.m_block_size] = acc_scale[j]+ if const_expr(self.p_scale_two_level):+ sScale[row_base + j + stage * self.m_block_size + self.m_block_size * 2] = 1.0+ cute.arch.mbarrier_arrive(+ mbar_ptr + self.mbar_softmax_corr_full_offset + stage+ )++ max_scaled = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ for j in cutlass.range_constexpr(rows):+ max_scaled[j] = row_max_safe[j] * softmax_scale_log2+ if const_expr(self.p_scale_rescale_log2):+ max_scaled[j] -= Float32(self.p_scale_rescale_log2)+ tSrS[j] = cute.arch.exp2(tSrS[j] * softmax_scale_log2 - max_scaled[j])+ row_sum[j] = row_sum[j] * acc_scale[j] + tSrS[j]++ tSrP = cute.make_rmem_tensor(frg_shape, cutlass.BFloat16)+ tSrP.store(tSrS.load().to(cutlass.BFloat16))+ for j in cutlass.range_constexpr(rows):+ sP_bf16_mk[row_base + j, tidx] = tSrP[j]+ cute.arch.fence_view_async_shared()+ cute.arch.mbarrier_arrive(+ mbar_ptr + self.mbar_bf16_P_full_offset+ )+ cute.arch.mbarrier_arrive(+ mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage+ )+ return mma_si_consumer_phase ^ 1, si_corr_producer_phase, red_buf++ @cute.jit+ def softmax_loop_transposed(+ self,+ stage: int | Int32,+ softmax_scale_log2: Float32,+ thr_mma_qk: cute.ThrMma,+ tStSi: cute.Tensor,+ sScale: cute.Tensor,+ mLSE: Optional[cute.Tensor],+ learnable_sink: Optional[cute.Tensor],+ mbar_ptr: cute.Pointer,+ block_info: BlockInfo,+ num_splits: Int32,+ SeqlenInfoCls: Callable,+ TileSchedulerCls: Callable,+ sSFP: cute.Tensor,+ sP_uint8: cute.Tensor,+ sSoftmaxRed: cute.Tensor,+ tidx: Int32,+ row_group: cutlass.Constexpr[int] = 0,+ mResidualSeqUsedK: Optional[cute.Tensor] = None,+ mSinkSeqUsedK: Optional[cute.Tensor] = None,+ sP_bf16: Optional[cute.Tensor] = None,+ sP_bf16_mk: Optional[cute.Tensor] = None,+ role: cutlass.Constexpr = SplitRole.ALL,+ ):+ """Softmax over an S tile held as (kv position, query row).++ Thread ``tidx`` owns kv position ``tidx`` of the block and the+ ``transposed_query_rows`` columns that carry a query, so it runs that+ many exponentials instead of a whole row of keys. The row maximum is+ along kv and therefore crosses threads; the row sum is deferred to the+ end of the tile because the running correction factor is the same in+ every thread and factors out of the accumulation.+ """+ runs_pages = const_expr(role != SplitRole.BF16)+ runs_tail = const_expr(+ self.fused_residual_first_block and role != SplitRole.PAGES+ )+ runs_sink = const_expr(self.fused_sink_block and role != SplitRole.PAGES)+ rows = const_expr(self.softmax_rows_per_group)+ row_base = const_expr(row_group * rows)+ # Only the leading columns of S carry a query; the rest of the MMA tile+ # is padding that the tensor-memory load simply never reads. A column of+ # S is a tensor-memory column, so a warpgroup reaches its own rows by+ # advancing the iterator past the group before it.+ tStS_group = cute.make_tensor(tStSi.iterator + row_base, tStSi.layout)+ tStS_live = cute.composition(tStS_group, cute.make_layout((self.m_block_size, rows)))+ tScS = thr_mma_qk.partition_C(cute.make_identity_tensor(self.mma_tiler_qk[:2]))+ tScS_live = cute.composition(tScS, cute.make_layout((self.m_block_size, rows)))+ tmem_load_atom = cute.make_copy_atom(+ tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(rows)),+ Float32,+ )+ thr_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tStS_live).get_slice(tidx)+ tStS_t2r = thr_tmem_load.partition_S(tStS_live)+ tStS_second_t2r = None+ if const_expr(self.stacked_query_terms):+ tStS_second_group = cute.make_tensor(+ tStSi.iterator + self.transposed_query_rows + row_base,+ tStSi.layout,+ )+ tStS_second_live = cute.composition(+ tStS_second_group, cute.make_layout((self.m_block_size, rows))+ )+ tStS_second_t2r = thr_tmem_load.partition_S(tStS_second_live)+ tStS_sink_t2r = None+ if const_expr(self.fused_sink_block):+ sink_offset = self.tmem_s_offset[1] - self.tmem_s_offset[0]+ tStS_sink_group = cute.make_tensor(+ tStSi.iterator + sink_offset + row_base, tStSi.layout+ )+ tStS_sink_live = cute.composition(+ tStS_sink_group,+ cute.make_layout((self.m_block_size, rows)),+ )+ tStS_sink_t2r = thr_tmem_load.partition_S(tStS_sink_live)+ frg_shape = thr_tmem_load.partition_D(tScS_live).shape++ if const_expr(runs_pages):+ self.zero_transposed_p(sP_uint8, sSFP, tidx)++ mma_si_consumer_phase = Int32(0)+ si_corr_producer_phase = Int32(1)+ red_buf = Int32(0)++ # The first FP4 QK may overwrite S once this arrives. Behind a tail+ # with a sink that is only after the tail's S has been read, so the+ # residual step arrives instead.+ if const_expr(runs_pages and not (runs_tail and self.fused_sink_block)):+ if stage == 1:+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_sfqk_load_offset + 0)+ if const_expr(self.q_stage == 1) and stage == 0:+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_sfqk_load_offset + 0)++ row_max = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ row_sum = cute.make_rmem_tensor(cute.make_layout(rows), Float32)++ softmax_step = partial(+ self.softmax_step_transposed,+ softmax_scale_log2=softmax_scale_log2,+ mbar_ptr=mbar_ptr,+ thr_tmem_load=thr_tmem_load,+ tStS_t2r=tStS_t2r,+ tStS_second_t2r=tStS_second_t2r,+ frg_shape=frg_shape,+ row_max=row_max,+ row_sum=row_sum,+ sScale=sScale,+ sSFP=sSFP,+ sP_uint8=sP_uint8,+ sSoftmaxRed=sSoftmaxRed,+ stage=stage,+ tidx=tidx,+ row_group=row_group,+ )++ tile_scheduler = TileSchedulerCls()+ work_tile = tile_scheduler.initial_work_tile_info()+ while work_tile.is_valid_tile:+ m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx+ iket.range_push("sm_seqinfo")+ seqlen = SeqlenInfoCls(batch_idx)+ n_block_min, n_block_max = self.split_block_range(+ block_info, seqlen, m_block, split_idx, num_splits+ )+ iket.range_pop()+ # A split can be handed no kv block at all. Such a tile must stay out+ # of the correction handshake entirely, as on the untransposed path,+ # or the two warp groups fall out of step.+ has_work = self.tile_has_work(+ n_block_min, n_block_max, split_idx, num_splits+ )+ if has_work:+ row_max.fill(-Float32.inf)+ row_sum.fill(0.0)++ iket.range_push("sm_wait_corr")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_softmax_corr_empty_offset + stage,+ si_corr_producer_phase,+ )+ iket.range_pop()+ si_corr_producer_phase ^= 1++ seeded = runs_tail+ if const_expr(runs_tail):+ if const_expr(self.fused_sink_block):+ red_buf = self.softmax_residual_step_transposed(+ softmax_scale_log2=softmax_scale_log2,+ mbar_ptr=mbar_ptr,+ thr_tmem_load=thr_tmem_load,+ tStS_t2r=tStS_t2r,+ frg_shape=frg_shape,+ row_max=row_max,+ row_sum=row_sum,+ sP_bf16=sP_bf16,+ sP_bf16_mk=sP_bf16_mk,+ sSoftmaxRed=sSoftmaxRed,+ red_buf=red_buf,+ tidx=tidx,+ batch_idx=batch_idx,+ mSinkSeqUsedK=mSinkSeqUsedK,+ split_idx=split_idx,+ mResidualSeqUsedK=mResidualSeqUsedK,+ seqused_fp4=seqlen.seqlen_k,+ row_group=row_group,+ pages_follow=runs_pages,+ )+ else:+ red_buf = self.softmax_residual_step_transposed(+ softmax_scale_log2=softmax_scale_log2,+ mbar_ptr=mbar_ptr,+ thr_tmem_load=thr_tmem_load,+ tStS_t2r=tStS_t2r,+ frg_shape=frg_shape,+ row_max=row_max,+ row_sum=row_sum,+ sP_bf16=sP_bf16,+ sP_bf16_mk=sP_bf16_mk,+ sSoftmaxRed=sSoftmaxRed,+ red_buf=red_buf,+ tidx=tidx,+ batch_idx=batch_idx,+ split_idx=split_idx,+ mResidualSeqUsedK=mResidualSeqUsedK,+ row_group=row_group,+ )++ if const_expr(runs_pages):+ sink_length = Int32(0)+ if const_expr(self.fused_sink_block):+ sink_length = mSinkSeqUsedK[batch_idx]+ mma_si_consumer_phase, si_corr_producer_phase, red_buf = softmax_step(+ mma_si_consumer_phase,+ si_corr_producer_phase,+ red_buf,+ n_block_max - 1,+ seqlen_k=seqlen.seqlen_k,+ is_first=not seeded,+ mask_seqlen=True,+ sink_length=sink_length,+ mask_sink=True,+ waits_for_previous=False,+ )+ # Blocks run from the last page down, so page zero, whose+ # sink duplicate is masked, can only be the first block or+ # the last; the steps between carry no sink test.+ peel_last = const_expr(self.fused_sink_block)+ for n_tile in cutlass.range(+ n_block_max - 1 - n_block_min - int(peel_last), unroll=1+ ):+ mma_si_consumer_phase, si_corr_producer_phase, red_buf = softmax_step(+ mma_si_consumer_phase,+ si_corr_producer_phase,+ red_buf,+ n_block_max - 2 - n_tile,+ seqlen_k=seqlen.seqlen_k,+ is_first=False,+ mask_seqlen=False,+ sink_length=sink_length,+ follows_pv=True,+ )+ if const_expr(peel_last):+ if n_block_max - 1 > n_block_min:+ mma_si_consumer_phase, si_corr_producer_phase, red_buf = softmax_step(+ mma_si_consumer_phase,+ si_corr_producer_phase,+ red_buf,+ n_block_min,+ seqlen_k=seqlen.seqlen_k,+ is_first=False,+ mask_seqlen=False,+ sink_length=sink_length,+ follows_pv=True,+ mask_sink=True,+ )++ if const_expr(runs_sink):+ (+ mma_si_consumer_phase,+ si_corr_producer_phase,+ red_buf,+ ) = self.softmax_sink_step_transposed(+ mma_si_consumer_phase=mma_si_consumer_phase,+ si_corr_producer_phase=si_corr_producer_phase,+ red_buf=red_buf,+ softmax_scale_log2=softmax_scale_log2,+ mbar_ptr=mbar_ptr,+ thr_tmem_load=thr_tmem_load,+ tStS_t2r=tStS_t2r,+ tStS_sink_t2r=tStS_sink_t2r,+ frg_shape=frg_shape,+ row_max=row_max,+ row_sum=row_sum,+ sScale=sScale,+ sP_bf16=sP_bf16,+ sP_bf16_mk=sP_bf16_mk,+ sSoftmaxRed=sSoftmaxRed,+ stage=stage,+ tidx=tidx,+ batch_idx=batch_idx,+ mSinkSeqUsedK=mSinkSeqUsedK,+ split_idx=split_idx,+ row_group=row_group,+ waits_for_previous=runs_pages,+ )++ red_buf = self.softmax_warpgroup_reduce(+ row_sum,+ sSoftmaxRed,+ tidx,+ red_buf,+ lambda a, b: a + b,+ row_group=row_group,+ )+ if const_expr(runs_pages or runs_sink):+ si_corr_producer_phase = self.softmax_wait_corr(+ mbar_ptr, stage, si_corr_producer_phase+ )+ if tidx == 0:+ for j in cutlass.range_constexpr(rows):+ sScale[row_base + j + stage * self.m_block_size] = row_sum[j]+ if const_expr(mLSE is not None or learnable_sink is not None):+ for j in cutlass.range_constexpr(rows):+ sScale[+ row_base + j + stage * self.m_block_size + self.m_block_size * 2+ ] = row_max[j]+ cute.arch.mbarrier_arrive(+ mbar_ptr + self.mbar_softmax_corr_full_offset + stage+ )++ tile_scheduler.advance_to_next_work()+ work_tile = tile_scheduler.get_current_work()++ @cute.jit+ def softmax_step_transposed(+ self,+ mma_si_consumer_phase: Int32,+ si_corr_producer_phase: Int32,+ red_buf: Int32,+ n_block: Int32,+ softmax_scale_log2: Float32,+ mbar_ptr: cute.Pointer,+ thr_tmem_load: cute.CopyAtom,+ tStS_t2r: cute.Tensor,+ tStS_second_t2r: Optional[cute.Tensor],+ frg_shape,+ row_max: cute.Tensor,+ row_sum: cute.Tensor,+ sScale: cute.Tensor,+ sSFP: cute.Tensor,+ sP_uint8: cute.Tensor,+ sSoftmaxRed: cute.Tensor,+ stage: int | Int32,+ tidx: Int32,+ seqlen_k: Int32,+ is_first: cutlass.Constexpr[bool],+ mask_seqlen: cutlass.Constexpr[bool],+ sink_length: Int32,+ row_group: cutlass.Constexpr[int] = 0,+ follows_pv: cutlass.Constexpr[bool] = False,+ mask_sink: cutlass.Constexpr[bool] = False,+ waits_for_previous: cutlass.Constexpr[bool] = True,+ ) -> Tuple[Int32, Int32, Int32]:+ """One KV block of the transposed softmax.++ The tensor-memory load gives thread ``tidx`` kv position ``tidx``, which+ is the invariant the whole step rests on: masking is one predicate, a+ scale group of 16 kv positions is 16 lanes of one warp, and the two FP4+ nibbles that share a byte sit in neighbouring lanes.++ ``follows_pv`` is every block of a tile but its first: the previous+ block's PV may still be reading the P tile this step rewrites.+ ``mask_sink`` is a block that may be page zero. The test sits on the+ softmax's critical path, which paces the whole loop, so the steps that+ cannot reach page zero leave it out.++ ``waits_for_previous`` is every step but the tile's first to publish:+ correction must have taken the previous step's factors from sScale+ before this step overwrites them. The wait sits just before the+ overwrite rather than at the end of the step that published, so+ correction has this step's row maximum to wake in.+ """+ rows = const_expr(self.softmax_rows_per_group)+ row_base = const_expr(row_group * rows)+ bytes_per_row = const_expr(self.n_block_size // 2)+ atom_k_bytes = const_expr(self.n_block_size // 4)++ iket.range_push("sm_wait_s")+ cute.arch.mbarrier_wait(mbar_ptr + self.mbar_S_full_offset + stage, mma_si_consumer_phase)+ iket.range_pop()+ tSrS = cute.make_rmem_tensor(frg_shape, self.qk_acc_dtype)+ cute.copy(thr_tmem_load, tStS_t2r, tSrS)+ if const_expr(self.stacked_query_terms):+ tSrS_second = cute.make_rmem_tensor(frg_shape, self.qk_acc_dtype)+ cute.copy(thr_tmem_load, tStS_second_t2r, tSrS_second)+ cute.arch.fence_view_async_tmem_load()+ cute.arch.mbarrier_arrive(+ mbar_ptr + self.mbar_sfqk_load_offset + (self.q_stage - 1 - stage)+ )+ if const_expr(self.stacked_query_terms):+ for j in cutlass.range_constexpr(rows):+ tSrS[j] = tSrS[j] + tSrS_second[j]++ if const_expr(mask_seqlen):+ # A row whose fp4 length is zero still runs one step, at block -1.+ # Folding that back onto block 0 puts the limit at the start of the+ # sequence, so every position is masked, as the untransposed mask+ # does for the same edge.+ n_block_masked = n_block+ if n_block_masked < 0:+ n_block_masked = Int32(0)+ if n_block_masked * self.n_block_size + tidx >= seqlen_k:+ for j in cutlass.range_constexpr(rows):+ tSrS[j] = -Float32.inf+ if const_expr(self.fused_sink_block and mask_sink):+ # Page zero retains a quantized duplicate of the permanent BF16+ # sink so that every later page keeps its original vLLM boundary.+ # Suppress only that prefix; all other FP4 positions are unchanged.+ if (n_block == Int32(0)) and (tidx < sink_length):+ for j in cutlass.range_constexpr(rows):+ tSrS[j] = -Float32.inf++ iket.range_push("sm_rowmax")+ # A scale group is sf_vec_size consecutive kv positions, hence that many+ # consecutive lanes, so the butterfly passes through the group maximum+ # on its way to the warp maximum. That intermediate is what the FP4+ # packing below needs, because exp2 is monotonic: the largest+ # exponential of a group is the exponential of the group's largest+ # score. Carrying it out costs one exp2 per row and removes a second+ # butterfly of the same depth.+ group_steps = const_expr(self.sf_vec_size.bit_length() - 1)+ warp_steps = const_expr(cute.arch.WARP_SIZE.bit_length() - 1)+ block_max = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ group_max_score = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ scatter = const_expr(self.softmax_scatters_rows)+ # Past the first block a row's maximum only moves when it grows by+ # more than the rescale threshold, which once it settles it almost+ # never does. Each thread applies the skip test below to its own+ # scores and one barrier ORs the verdicts. The test is monotonic in+ # the score, so it passes on the row maximum exactly when it passes on+ # every score of the row, and the cross-warp maximum is reduced only+ # when some row will take it.+ votes = const_expr(not is_first and RESCALE_THRESHOLD > 0.0 and not self.p_scale_two_level)+ grows = cutlass.Boolean(True)+ if const_expr(votes):+ own_grows = Int32(0)+ for j in cutlass.range_constexpr(rows):+ grown = (+ row_max[j] - cute.arch.fmax(row_max[j], tSrS[j])+ ) * softmax_scale_log2+ own_grows = Int32(+ cutlass.select_(grown >= -RESCALE_THRESHOLD, own_grows, Int32(1))+ )+ # Every thread gets the same verdict back; voting on it tells the+ # compiler so, which keeps divergence handling out of the branches+ # it decides.+ grows = cute.arch.vote_any_sync(+ utils.barrier_or(+ own_grows,+ barrier_id=int(NamedBarrierFwd.SoftmaxReduce) + row_group,+ number_of_threads=len(self.softmax0_warp_ids) * cute.arch.WARP_SIZE,+ )+ != Int32(0)+ )+ if const_expr(scatter):+ # Reduce-scatter: each of the first log2(rows) butterfly steps hands+ # half of the rows still held to the partner lane and keeps the+ # other half, so a lane ends with one row, the one ``scatter_row``+ # names, and the steps cost rows - 1 shuffles in all rather than+ # one per row each. The rest of the butterfly then runs on that+ # row alone, through its group maximum to its warp maximum.+ scatter_steps = const_expr(rows.bit_length() - 1)+ held = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ for j in cutlass.range_constexpr(rows):+ held[j] = tSrS[j]+ for i in cutlass.range_constexpr(scatter_steps):+ half = const_expr(rows >> (i + 1))+ upper = ((tidx >> i) & 1) == 1+ for k in cutlass.range_constexpr(half):+ send = Float32(cutlass.select_(upper, held[k], held[k + half]))+ keep = Float32(cutlass.select_(upper, held[k + half], held[k]))+ held[k] = cute.arch.fmax(+ keep, cute.arch.shuffle_sync_bfly(send, offset=1 << i)+ )+ val = held[0]+ for i in cutlass.range_constexpr(scatter_steps, group_steps):+ val = cute.arch.fmax(val, cute.arch.shuffle_sync_bfly(val, offset=1 << i))+ own_group_max_score = val+ own_row = self.scatter_row(tidx)+ if grows:+ for i in cutlass.range_constexpr(group_steps, warp_steps):+ val = cute.arch.fmax(+ val, cute.arch.shuffle_sync_bfly(val, offset=1 << i)+ )+ red_buf = self.softmax_warpgroup_reduce_scattered(+ val,+ own_row,+ block_max,+ sSoftmaxRed,+ tidx,+ red_buf,+ row_group=row_group,+ )+ else:+ for j in cutlass.range_constexpr(rows):+ val = tSrS[j]+ for i in cutlass.range_constexpr(group_steps):+ val = cute.arch.fmax(val, cute.arch.shuffle_sync_bfly(val, offset=1 << i))+ group_max_score[j] = val+ block_max[j] = val+ if grows:+ for j in cutlass.range_constexpr(rows):+ val = block_max[j]+ for i in cutlass.range_constexpr(group_steps, warp_steps):+ val = cute.arch.fmax(+ val, cute.arch.shuffle_sync_bfly(val, offset=1 << i)+ )+ block_max[j] = val+ red_buf = self.softmax_warpgroup_reduce(+ block_max,+ sSoftmaxRed,+ tidx,+ red_buf,+ cute.arch.fmax,+ intra_warp=False,+ row_group=row_group,+ )+ acc_scale = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ row_max_safe = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ # Once the maximum settles every row skips its rescale, and the skip+ # needs only the vote: the factor is 1 and the maximum stays. A block+ # where some row grows takes the full update below, which applies the+ # same test row by row against the reduced maximum.+ if not grows:+ for j in cutlass.range_constexpr(rows):+ row_max_safe[j] = row_max[j]+ acc_scale[j] = Float32(1.0)+ else:+ for j in cutlass.range_constexpr(rows):+ if const_expr(is_first):+ row_max_new = block_max[j]+ row_max_safe[j] = row_max_new if row_max_new != -Float32.inf else 0.0+ acc_scale[j] = 0.0+ else:+ row_max_old = row_max[j]+ row_max_new = cute.arch.fmax(row_max_old, block_max[j])+ safe = row_max_new if row_max_new != -Float32.inf else 0.0+ acc_scale_ = (row_max_old - safe) * softmax_scale_log2+ # The skip decision is made on a maximum every thread agrees+ # on, so it stays uniform across the warpgroup, and a+ # skipped rescale costs no exponential.+ scale = Float32(1.0)+ if const_expr(RESCALE_THRESHOLD > 0.0):+ if acc_scale_ >= -RESCALE_THRESHOLD:+ row_max_new = row_max_old+ safe = row_max_old+ else:+ scale = utils.exp2f(acc_scale_)+ else:+ scale = utils.exp2f(acc_scale_)+ row_max_safe[j] = safe+ acc_scale[j] = scale+ row_max[j] = row_max_new+ # The shift is the same for every score of a row, so scaling it once+ # leaves each exponential a single fused multiply-add. A reduce-+ # scattered tile holds an even number of rows, so its arithmetic runs+ # on pairs of rows in the packed FP32x2 units; one rounding per lane,+ # as the scalar instructions round.+ max_scaled = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ neg_max_scaled = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ if const_expr(scatter):+ for j in cutlass.range_constexpr(0, rows, 2):+ max_scaled[j], max_scaled[j + 1] = utils.mul_packed_f32x2(+ (row_max_safe[j], row_max_safe[j + 1]),+ (softmax_scale_log2, softmax_scale_log2),+ )+ neg_max_scaled[j], neg_max_scaled[j + 1] = utils.mul_packed_f32x2(+ (row_max_safe[j], row_max_safe[j + 1]),+ (-softmax_scale_log2, -softmax_scale_log2),+ )+ else:+ for j in cutlass.range_constexpr(rows):+ max_scaled[j] = row_max_safe[j] * softmax_scale_log2+ if const_expr(self.p_scale_rescale_log2):+ for j in cutlass.range_constexpr(rows):+ max_scaled[j] -= Float32(self.p_scale_rescale_log2)+ if const_expr(scatter):+ neg_max_scaled[j] += Float32(self.p_scale_rescale_log2)+ iket.range_pop()++ if const_expr(waits_for_previous):+ si_corr_producer_phase = self.softmax_wait_corr(+ mbar_ptr, stage, si_corr_producer_phase+ )+ # Every thread holds the same correction factors, so one publishes them.+ if const_expr(not is_first):+ if tidx == 0:+ for j in cutlass.range_constexpr(rows):+ sScale[row_base + j + stage * self.m_block_size] = acc_scale[j]+ if const_expr(self.p_scale_two_level):+ if tidx == 0:+ for j in cutlass.range_constexpr(rows):+ outer = cute.arch.exp2(+ block_max[j] * softmax_scale_log2 - max_scaled[j]+ - Float32(math.log2(448.0 * 6.0))+ )+ sScale[row_base + j + stage * self.m_block_size + self.m_block_size * 2] = outer+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_full_offset + stage)++ amplified = None+ if const_expr(self.p_scale_two_level):+ amplified = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ for j in cutlass.range_constexpr(rows):+ peak = Float32(cutlass.select_(block_max[j] == -Float32.inf, 0.0, block_max[j]))+ amplified[j] = cute.arch.exp2(+ (tSrS[j] - peak) * softmax_scale_log2+ + Float32(math.log2(448.0 * 6.0))+ )+ iket.range_push("sm_exp")+ if const_expr(scatter):+ for j in cutlass.range_constexpr(0, rows, 2):+ arg0, arg1 = utils.fma_packed_f32x2(+ (tSrS[j], tSrS[j + 1]),+ (softmax_scale_log2, softmax_scale_log2),+ (neg_max_scaled[j], neg_max_scaled[j + 1]),+ )+ tSrS[j] = cute.arch.exp2(arg0)+ tSrS[j + 1] = cute.arch.exp2(arg1)+ if const_expr(is_first):+ row_sum[j] = tSrS[j]+ row_sum[j + 1] = tSrS[j + 1]+ else:+ row_sum[j], row_sum[j + 1] = utils.fma_packed_f32x2(+ (row_sum[j], row_sum[j + 1]),+ (acc_scale[j], acc_scale[j + 1]),+ (tSrS[j], tSrS[j + 1]),+ )+ else:+ for j in cutlass.range_constexpr(rows):+ tSrS[j] = cute.arch.exp2(tSrS[j] * softmax_scale_log2 - max_scaled[j])+ if const_expr(is_first):+ row_sum[j] = tSrS[j]+ else:+ row_sum[j] = row_sum[j] * acc_scale[j] + tSrS[j]+ iket.range_pop()++ if const_expr(self.p_scale_two_level):+ for j in cutlass.range_constexpr(rows):+ peak = Float32(cutlass.select_(block_max[j] == -Float32.inf, 0.0, block_max[j]))+ max_scaled[j] = peak * softmax_scale_log2 - Float32(math.log2(448.0 * 6.0))+ tSrS[j] = amplified[j]+ iket.range_push("sm_pquant")+ group_max = cute.make_rmem_tensor(cute.make_layout(rows), Float32)+ group_exponent = cute.make_rmem_tensor(cute.make_layout(rows), Int32)+ own_scale = Float32(0.0)+ own_exponent = Int32(0)+ if const_expr(scatter):+ # Every lane of a scale group would compute the same scale for a+ # row, so the lane that holds the row's group maximum computes it+ # once and the group's lanes fetch its reciprocal from there.+ own_max_scaled = max_scaled[0]+ for j in cutlass.range_constexpr(1, rows):+ own_max_scaled = Float32(+ cutlass.select_(own_row == j, max_scaled[j], own_max_scaled)+ )+ maximum = cute.arch.exp2(+ own_group_max_score * softmax_scale_log2 - own_max_scaled+ )+ if const_expr(self.is_mxfp4):+ own_exponent = round_up_exponent(maximum)+ else:+ if const_expr(self.p_scale_amax2_or_amax6):+ own_scale = _amax2_or_amax6_probability_scale(maximum)+ else:+ own_scale = maximum * Float32(1.0 / self.p_scale_divisor)+ own_inverse = cute.arch.rcp_approx(cute.arch.fmax(own_scale, 1e-20))+ cluster = tidx & (cute.arch.WARP_SIZE - rows)+ if const_expr(self.is_mxfp4):+ # Each row still has one P value in every lane. Preserve the+ # scattered maximum/exponential arithmetic, but reduce each+ # candidate's error over all 32 values of that row's group.+ for j in cutlass.range_constexpr(rows):+ exponent = Int32(cute.arch.shuffle_sync(+ own_exponent, cluster | self.scatter_lane(j)+ ))+ group_exponent[j] = _mxfp4_probability_exponent(tSrS[j], exponent)+ tSrS[j] = tSrS[j] * inverse_power_of_two(group_exponent[j])+ # Do not overwrite any scatter owner until all original+ # exponents have been fetched by every row above.+ for j in cutlass.range_constexpr(rows):+ own_exponent = Int32(cutlass.select_(+ own_row == j, group_exponent[j], own_exponent+ ))+ else:+ for j in cutlass.range_constexpr(0, rows, 2):+ tSrS[j], tSrS[j + 1] = utils.mul_packed_f32x2(+ (tSrS[j], tSrS[j + 1]),+ (+ cute.arch.shuffle_sync(own_inverse, cluster | self.scatter_lane(j)),+ cute.arch.shuffle_sync(+ own_inverse, cluster | self.scatter_lane(j + 1)+ ),+ ),+ )+ for j in cutlass.range_constexpr(0 if scatter else rows):+ # Same expression as the exponentials above, so this is the value+ # the largest lane of the group holds, to the bit.+ maximum = cute.arch.exp2(+ group_max_score[j] * softmax_scale_log2 - max_scaled[j]+ )+ if const_expr(self.is_mxfp4):+ group_exponent[j] = _mxfp4_probability_exponent(+ tSrS[j], round_up_exponent(maximum)+ )+ elif const_expr(self.p_scale_amax2_or_amax6):+ group_max[j] = _amax2_or_amax6_probability_scale(maximum)+ elif const_expr(self.p_scale_grid_search):+ best_scale = maximum * Float32(1.0 / 6.0)+ best_stored = _round_probability_scale(best_scale)+ best_code = _round_probability_code(+ tSrS[j]+ * cute.arch.rcp_approx(+ cute.arch.fmax(best_scale, 1e-20)+ )+ )+ difference = best_code * best_stored - tSrS[j]+ best_error = difference * difference+ for i in cutlass.range_constexpr(group_steps):+ best_error += cute.arch.shuffle_sync_bfly(+ best_error, offset=1 << i+ )+ rival_divisors = (4.0, 3.0, 2.0)+ for candidate in cutlass.range_constexpr(+ len(rival_divisors)+ ):+ divisor = rival_divisors[candidate]+ rival_scale = maximum * Float32(1.0 / divisor)+ rival_stored = _round_probability_scale(rival_scale)+ rival_code = _round_probability_code(+ tSrS[j]+ * cute.arch.rcp_approx(+ cute.arch.fmax(rival_scale, 1e-20)+ )+ )+ difference = rival_code * rival_stored - tSrS[j]+ rival_error = difference * difference+ for i in cutlass.range_constexpr(group_steps):+ rival_error += cute.arch.shuffle_sync_bfly(+ rival_error, offset=1 << i+ )+ better = rival_error < best_error+ best_scale = Float32(+ cutlass.select_(better, rival_scale, best_scale)+ )+ best_error = Float32(+ cutlass.select_(better, rival_error, best_error)+ )+ group_max[j] = best_scale+ else:+ group_max[j] = maximum * Float32(+ 1.0 / self.p_scale_divisor+ )+ for j in cutlass.range_constexpr(0 if scatter else rows):+ # An approximate reciprocal is one instruction where the exact+ # division is a sequence, and P carries three mantissa bits.+ if const_expr(self.is_mxfp4):+ tSrS[j] = tSrS[j] * inverse_power_of_two(group_exponent[j])+ else:+ tSrS[j] = tSrS[j] * cute.arch.rcp_approx(cute.arch.fmax(group_max[j], 1e-20))++ # The S this step read was signalled before the previous block's PV+ # was even issued, so that PV is waited for here, as late as possible.+ # It completes once per block and runs at most one block behind, so+ # the S phase of the block before names it.+ if const_expr(follows_pv):+ iket.range_push("sm_wait_pv")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_P_consumed_offset + stage,+ mma_si_consumer_phase ^ 1,+ )+ iket.range_pop()++ # K-major P wants kv contiguous, so the nibbles for kv = tidx and+ # tidx + 1 share a byte. One butterfly per query row brings the odd+ # lane's value over and the even lane stores the merged byte.+ k_byte = tidx // 2+ for j in cutlass.range_constexpr(rows):+ row = row_base + j+ partner = cute.arch.shuffle_sync_bfly(tSrS[j], offset=1)+ if tidx % 2 == 0:+ packed = sm100_utils.packed_float_to_e2m1x2(partner, tSrS[j])+ sP_uint8[(row, k_byte % atom_k_bytes), 0, k_byte // atom_k_bytes, 0] = (+ cutlass.Uint8(packed & 0xFF)+ )++ sSFP_u8 = cute.recast_tensor(sSFP[None, None, None, stage], cutlass.Uint8)+ if const_expr(scatter):+ # The first lanes of a group hold one row each, so each stores the+ # scale of its own row.+ if tidx % self.sf_vec_size < rows:+ k_group = tidx // self.sf_vec_size+ row = row_base + own_row+ if const_expr(self.is_mxfp4):+ sf_byte = own_exponent + Int32(127)+ else:+ sf_byte = packed_float_to_ue4m3(+ own_scale, Float32(0.0), Float32(0.0), Float32(0.0)+ )+ sSFP_u8[+ (((row % 32, row // 32), 0),+ (0, k_group % self.sf_groups_per_mma_k)),+ 0, k_group // self.sf_groups_per_mma_k+ ] = cutlass.Uint8(sf_byte & 0xFF)+ elif tidx % self.sf_vec_size == 0:+ k_group = tidx // self.sf_vec_size+ for j in cutlass.range_constexpr(rows):+ row = row_base + j+ if const_expr(self.is_mxfp4):+ sf_byte = group_exponent[j] + Int32(127)+ else:+ sf_byte = packed_float_to_ue4m3(+ group_max[j], Float32(0.0), Float32(0.0), Float32(0.0)+ )+ sSFP_u8[+ (((row % 32, row // 32), 0),+ (0, k_group % self.sf_groups_per_mma_k)),+ 0, k_group // self.sf_groups_per_mma_k+ ] = cutlass.Uint8(sf_byte & 0xFF)+ # The MMA reads P and its scale factors through the async proxy while+ # these are generic-proxy stores; the mbarrier below only orders generic+ # traffic, so the fence is what makes them visible.+ cute.arch.fence_view_async_shared()+ iket.range_pop()++ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage)+ return mma_si_consumer_phase ^ 1, si_corr_producer_phase, red_buf++ # for both softmax0 and softmax1 warp group+ @cute.jit+ def softmax_loop(+ self,+ stage: int | Int32,+ softmax_scale_log2: Float32,+ softmax_scale: Float32,+ thr_mma_qk: cute.ThrMma,+ tStSi: cute.Tensor,+ sScale: cute.Tensor,+ mLSE: Optional[cute.Tensor],+ learnable_sink: Optional[cute.Tensor],+ mbar_ptr: cute.Pointer,+ block_info: BlockInfo,+ num_splits: Int32,+ SeqlenInfoCls: Callable,+ AttentionMaskCls: Callable,+ TileSchedulerCls: Callable,+ aux_tensors: Optional[list] = None,+ fastdiv_mods=(None, None),+ blocksparse_tensors: Optional[BlockSparseTensors] = None,+ tCtSFP: Optional[Tuple[cute.Tensor, ...]] = None,+ sSFP: Optional[cute.Tensor] = None,+ mResidualSeqUsedK: Optional[cute.Tensor] = None,+ mSinkSeqUsedK: Optional[cute.Tensor] = None,+ sP_uint8: Optional[cute.Tensor] = None,+ sSoftmaxRed: Optional[cute.Tensor] = None,+ sP_bf16: Optional[cute.Tensor] = None,+ sP_bf16_mk: Optional[cute.Tensor] = None,+ row_group: cutlass.Constexpr[int] = 0,+ role: cutlass.Constexpr = SplitRole.ALL,+ ):+ """Compute softmax on attention scores from QK matrix multiplication.++ This method handles the softmax computation for either the first or second half of the+ attention matrix, depending on the 'stage' parameter. It calculates row-wise maximum+ and sum values needed for stable softmax computation, applies optional masking, and+ transforms raw attention scores into probability distributions.++ The implementation uses specialized memory access patterns and efficient math operations+ for computing exp(x) using exp2 functions. It also coordinates pipeline+ synchronization between MMA, correction, and sequence processing stages.+ """+ tidx = cute.arch.thread_idx()[0] % (+ cute.arch.WARP_SIZE+ # * (len(self.softmax0_warp_ids) if stage == 0 else len(self.softmax1_warp_ids)+ * (len(self.softmax0_warp_ids))+ )++ tStScale = cute.composition(tStSi, cute.make_layout((self.m_block_size, 1)))+ tScS = thr_mma_qk.partition_C(cute.make_identity_tensor(self.mma_tiler_qk[:2]))+ tScScale = cute.composition(tScS, cute.make_layout((self.m_block_size, 1)))++ tilePlikeFP32 = self.mma_tiler_qk[1] // 32 * self.v_dtype.width+ tStP_layout = cute.composition(+ tStSi.layout, cute.make_layout((self.m_block_size, tilePlikeFP32))+ )+ tStP = cute.make_tensor(tStSi.iterator + self.tmem_s_to_p_offset, tStP_layout)++ tmem_load_atom = cute.make_copy_atom(+ tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(32)),+ Float32,+ )+ thr_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tStSi).get_slice(tidx)+ tStS_t2r = thr_tmem_load.partition_S(tStSi)+ tmem_store_scale_atom = cute.make_copy_atom(+ tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(1)),+ Float32,+ )+ thr_tmem_store_scale = tcgen05.make_tmem_copy(tmem_store_scale_atom, tStScale).get_slice(tidx)++ tStScale_r2t = thr_tmem_store_scale.partition_D(tStScale)+ tmem_store_atom = cute.make_copy_atom(+ tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(16 if const_expr(not self.quant_pv) else 8)),+ Float32,+ )+ thr_tmem_store = tcgen05.make_tmem_copy(tmem_store_atom, tStP).get_slice(tidx)+ tStP_r2t = thr_tmem_store.partition_D(tStP)++ if const_expr(self.transpose_s):+ self.softmax_loop_transposed(+ stage=stage,+ softmax_scale_log2=softmax_scale_log2,+ thr_mma_qk=thr_mma_qk,+ tStSi=tStSi,+ sScale=sScale,+ mLSE=mLSE,+ learnable_sink=learnable_sink,+ mbar_ptr=mbar_ptr,+ block_info=block_info,+ num_splits=num_splits,+ SeqlenInfoCls=SeqlenInfoCls,+ TileSchedulerCls=TileSchedulerCls,+ sSFP=sSFP,+ sP_uint8=sP_uint8,+ sSoftmaxRed=sSoftmaxRed,+ tidx=tidx,+ row_group=row_group,+ mResidualSeqUsedK=mResidualSeqUsedK,+ mSinkSeqUsedK=mSinkSeqUsedK,+ sP_bf16=sP_bf16,+ sP_bf16_mk=sP_bf16_mk,+ role=role,+ )+ return++ thr_tmem_store_bf16 = thr_tmem_store+ tStP_bf16_r2t = tStP_r2t+ if const_expr(self.fused_residual_first_block):+ tilePlikeFP32_bf16 = self.mma_tiler_qk[1] // 32 * 16 # BF16=16 bits+ tStP_bf16_layout = cute.composition(+ tStSi.layout, cute.make_layout((self.m_block_size, tilePlikeFP32_bf16))+ )+ tStP_bf16 = cute.make_tensor(+ tStSi.iterator + self.tmem_s_to_p_offset, tStP_bf16_layout+ )+ tmem_store_bf16_atom = cute.make_copy_atom(+ tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(16)),+ Float32,+ )+ thr_tmem_store_bf16 = tcgen05.make_tmem_copy(+ tmem_store_bf16_atom, tStP_bf16+ ).get_slice(tidx)+ tStP_bf16_r2t = thr_tmem_store_bf16.partition_D(tStP_bf16)++ mma_si_consumer_phase = Int32(0)+ si_corr_producer_phase = Int32(1)+ s0_s1_sequence_phase = Int32(1 if stage == 0 else 0)++ if stage == 1 and const_expr(self.quant_qk):+ sfqk_stage = 0+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_sfqk_load_offset + sfqk_stage)+ if const_expr(self.q_stage == 1) and stage == 0 and const_expr(self.quant_qk):+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_sfqk_load_offset + 0)+ # self.warp_scheduler_barrier_init()++ warp_idx_in_wg = cute.arch.make_warp_uniform(cute.arch.warp_idx()) % 4+ mbar_s0_s1_sequence_offset = self.mbar_s0_s1_sequence_offset + warp_idx_in_wg++ tile_scheduler = TileSchedulerCls()+ work_tile = tile_scheduler.initial_work_tile_info()+ while work_tile.is_valid_tile:+ m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx+ seqlen = SeqlenInfoCls(batch_idx)+ n_block_min, n_block_max = self.split_block_range(+ block_info, seqlen, m_block, split_idx, num_splits+ )++ mask = AttentionMaskCls(seqlen.seqlen_q, seqlen.seqlen_k)+ shared_mask_kwargs = dict(+ m_block=self.q_stage * m_block + stage,+ thr_mma=thr_mma_qk,+ thr_tmem_load=thr_tmem_load,+ mask_causal=self.is_causal,+ mask_local=self.is_local,+ batch_idx=batch_idx,+ head_idx=head_idx,+ aux_tensors=aux_tensors,+ )+ block_mask_mod = self.mask_mod if const_expr(self.use_block_sparsity) else None+ mask_fn = partial(+ mask.apply_mask_sm100,+ mask_mod=block_mask_mod,+ **shared_mask_kwargs,+ )+ if const_expr(self.use_block_sparsity):+ # Full blocks dont need mask_mod+ mask_fn_none = partial(+ mask.apply_mask_sm100,+ mask_mod=None,+ **shared_mask_kwargs,+ )+ else:+ mask_fn_none = None++ softmax = SoftmaxSm100.create(+ softmax_scale_log2,+ rescale_threshold=RESCALE_THRESHOLD,+ # rescale_threshold=8.0 if const_expr(self.q_dtype.width == 16) else 0.0, # (Wenxuan) disable skipping rescale until FP4 precision is verified+ softmax_scale=softmax_scale,+ quant_pv=self.quant_pv,+ compute_sp1=self.compute_sp1,+ kahan=self.fused_residual_first_block,+ p_scale_divisor=self.p_scale_divisor,+ p_scale_grid_search=self.p_scale_grid_search,+ p_scale_amax2_or_amax6=self.p_scale_amax2_or_amax6,+ )+ softmax.reset()++ if const_expr(self.use_block_sparsity):+ tile_block_count = get_total_block_count(blocksparse_tensors, batch_idx, head_idx, m_block)+ has_work = tile_block_count > Int32(0)+ else:+ tile_block_count = n_block_max - n_block_min+ has_work = self.tile_has_work(+ n_block_min, n_block_max, split_idx, num_splits+ )++ softmax_step = partial(+ self.softmax_step,+ softmax=softmax,+ mbar_ptr=mbar_ptr,+ mbar_s0_s1_sequence_offset=mbar_s0_s1_sequence_offset,+ thr_mma_qk=thr_mma_qk,+ thr_tmem_load=thr_tmem_load,+ thr_tmem_store=thr_tmem_store,+ thr_tmem_store_scale=thr_tmem_store_scale,+ tStS_t2r=tStS_t2r,+ tStScale_r2t=tStScale_r2t,+ tStP_r2t=tStP_r2t,+ sScale=sScale,+ stage=stage,+ batch_idx=batch_idx,+ head_idx=head_idx,+ m_block=self.q_stage * m_block + stage,+ seqlen=seqlen,+ aux_tensors=aux_tensors,+ fastdiv_mods=fastdiv_mods,+ mask_fn=partial(mask_fn, mask_seqlen=False),+ tCtSFP=tCtSFP,+ sSFP=sSFP,+ )++ if has_work:+ # Softmax acts as the producer: wait until correction signals the stage is empty+ iket.range_push("sm_wait_corr")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_softmax_corr_empty_offset + stage, si_corr_producer_phase+ )+ iket.range_pop()+ si_corr_producer_phase ^= 1++ if const_expr(self.fused_residual_first_block) and stage == 0 and has_work:+ mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = (+ self.softmax_residual_step(+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ softmax=softmax,+ mbar_ptr=mbar_ptr,+ thr_mma_qk=thr_mma_qk,+ thr_tmem_load=thr_tmem_load,+ thr_tmem_store_bf16=thr_tmem_store_bf16,+ thr_tmem_store_scale=thr_tmem_store_scale,+ tStS_t2r=tStS_t2r,+ tStP_bf16_r2t=tStP_bf16_r2t,+ sScale=sScale,+ stage=stage,+ batch_idx=batch_idx,+ m_block=self.q_stage * m_block + stage,+ mResidualSeqUsedK=mResidualSeqUsedK,+ thread_idx=tidx,+ split_idx=split_idx,+ )+ )++ # Block sparse or dense iteration+ if const_expr(self.use_block_sparsity):+ (+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ empty_tile,+ ) = softmax_block_sparse_sm100(+ blocksparse_tensors,+ batch_idx,+ head_idx,+ m_block,+ softmax_step,+ mask_fn,+ mask_fn_none,+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ mbar_ptr,+ self.mbar_softmax_corr_full_offset,+ self.mbar_softmax_corr_empty_offset,+ self.mbar_P_full_O_rescaled_offset,+ self.mbar_P_full_2_offset,+ self.q_stage,+ Int32(stage),+ )+ if not empty_tile:+ if const_expr(self.fused_residual_first_block):+ sScale[tidx + stage * self.m_block_size] = (+ softmax.row_sum[0] + softmax.row_sum_comp[0]+ )+ else:+ sScale[tidx + stage * self.m_block_size] = softmax.row_sum[0]+ if const_expr(mLSE is not None or learnable_sink is not None):+ sScale[+ tidx + stage * self.m_block_size + self.m_block_size * 2+ ] = softmax.row_max[0]+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_full_offset + stage)+ else:+ if has_work:+ if const_expr(self.fused_residual_first_block) and stage == 0:+ mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = softmax_step(+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ n_block_max - 1,+ is_first=False,+ mask_fn=partial(mask_fn, mask_seqlen=True),+ )+ else:+ mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = softmax_step(+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ n_block_max - 1,+ is_first=True,+ mask_fn=partial(mask_fn, mask_seqlen=True),+ )+ n_block_max -= 1+ # Next couple of iterations with causal masking+ if const_expr(self.is_causal or self.is_local):+ n_block_min_causal_local_mask = block_info.get_n_block_min_causal_local_mask(+ seqlen, m_block, n_block_min+ )+ for n_tile in cutlass.range(n_block_max - n_block_min_causal_local_mask, unroll=1):+ n_block = n_block_max - 1 - n_tile+ mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = (+ softmax_step(+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ n_block,+ mask_fn=partial(mask_fn, mask_seqlen=False),+ )+ )+ n_block_max = cutlass.min(n_block_max, n_block_min_causal_local_mask)+ # The remaining iterations have no masking+ n_block_min_before_local_mask = block_info.get_n_block_min_before_local_mask(+ seqlen, m_block, n_block_min+ )+ for n_tile in cutlass.range(n_block_max - n_block_min_before_local_mask, unroll=1):+ n_block = n_block_max - n_tile - 1+ mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = softmax_step(+ mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase, n_block+ )+ # Separate iterations with local masking on the left+ if const_expr(self.is_local and block_info.window_size_left is not None):+ n_block_max = cutlass.min(n_block_max, n_block_min_before_local_mask)+ for n_tile in cutlass.range(0, n_block_max - n_block_min, unroll=1):+ n_block = n_block_max - 1 - n_tile+ mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = (+ softmax_step(+ mma_si_consumer_phase,+ si_corr_producer_phase,+ s0_s1_sequence_phase,+ n_block,+ mask_fn=partial(mask_fn, mask_seqlen=False),+ )+ )+ # Now that we no longer already have the 1st iteration, need mask_seqlen=True here++ if const_expr(self.fused_residual_first_block):+ sScale[tidx + stage * self.m_block_size] = (+ softmax.row_sum[0] + softmax.row_sum_comp[0]+ )+ else:+ sScale[tidx + stage * self.m_block_size] = softmax.row_sum[0]+ if const_expr(mLSE is not None or learnable_sink is not None):+ sScale[+ tidx + stage * self.m_block_size + self.m_block_size * 2+ ] = softmax.row_max[0]+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_full_offset + stage)++ # # Write LSE to gmem+ # if const_expr(mLSE is not None):+ # acc_O_mn_row_is_zero_or_nan = softmax.row_sum[0] == 0.0 or softmax.row_sum[0] != softmax.row_sum[0]+ # scale = (+ # cute.arch.rcp_approx(softmax.row_sum[0] if not acc_O_mn_row_is_zero_or_nan else 1.0)+ # )+ # LN2 = math.log(2.0)+ # lse = (+ # (softmax.row_max[0] * softmax.scale_log2 + utils.log2f(softmax.row_sum[0])) * LN2+ # if not acc_O_mn_row_is_zero_or_nan else -Float32.inf+ # )+ # if const_expr(not seqlen.has_cu_seqlens_q):+ # mLSE_cur = mLSE[None, head_idx, batch_idx]+ # else:+ # mLSE_cur = cute.domain_offset((seqlen.offset_q,), mLSE[None, head_idx])+ # gLSE = cute.local_tile(mLSE_cur, (self.m_block_size,), (m_block * 2 + stage,))+ # if tidx < seqlen.seqlen_q - (m_block * 2 + stage) * self.m_block_size:+ # gLSE[tidx] = lse++ # Advance to next tile+ tile_scheduler.advance_to_next_work()+ work_tile = tile_scheduler.get_current_work()+ # End of persistent scheduler loop++ @cute.jit+ def _quant_fp4(self,+ # src+ tSrP_f32: cute.Tensor,+ tSrPSF_f32: cute.Tensor,+ # dst+ tSrP: cute.Tensor,+ tSrPSF: cute.Tensor,+ ):+ tSrP_f32_frag = cute.logical_divide(tSrP_f32, cute.make_layout(self.sf_vec_size))+ assert cute.size(tSrP_f32_frag, mode=[1]) == cute.size(tSrPSF_f32)+ tSrP_frag = cute.logical_divide(tSrP, cute.make_layout(self.sf_vec_size))+ tSrPSF_u32_view = cute.recast_tensor(tSrPSF, cute.Int32)++ # Process in groups of 4 for UE4M3 conversion+ assert cute.size(tSrPSF_f32) % 4 == 0+ for i in cutlass.range_constexpr(0, cute.size(tSrPSF_f32) // 4, unroll=1):+ # for i in cutlass.range_constexpr(0, 2):+ # Pack 4 FP32 values into UE4M3 format+ packed_ue4m3 = packed_float_to_ue4m3(+ tSrPSF_f32[i * 4],+ tSrPSF_f32[i * 4 + 1],+ tSrPSF_f32[i * 4 + 2],+ tSrPSF_f32[i * 4 + 3]+ )+ tSrPSF_u32_view[i] = packed_ue4m3++ # Quantize main tensor to E2M1 format (8 values per uint32_t)+ # Process in groups of 8 for E2M1 conversion+ for i in cutlass.range_constexpr(0, cute.size(tSrP_frag, mode=[1])):+ tSrP_u32_view = cute.recast_tensor(tSrP_frag[None, i], cute.Int32)+ for k in cutlass.range_constexpr(0, cute.size(tSrP_u32_view, mode=[0])):+ packed_e2m1 = packed_float_to_e2m1(+ tSrP_f32_frag[k * 8, i],+ tSrP_f32_frag[k * 8 + 1, i],+ tSrP_f32_frag[k * 8 + 2, i],+ tSrP_f32_frag[k * 8 + 3, i],+ tSrP_f32_frag[k * 8 + 4, i],+ tSrP_f32_frag[k * 8 + 5, i],+ tSrP_f32_frag[k * 8 + 6, i],+ tSrP_f32_frag[k * 8 + 7, i]+ )+ tSrP_u32_view[k] = packed_e2m1++ @cute.jit+ def softmax_step(+ self,+ mma_si_consumer_phase: Int32,+ si_corr_producer_phase: Int32,+ s0_s1_sequence_phase: Int32,+ n_block: Int32,+ softmax: SoftmaxSm100,+ mbar_ptr: cute.Pointer,+ mbar_s0_s1_sequence_offset: Int32,+ thr_mma_qk: cute.ThrMma,+ thr_tmem_load: cute.CopyAtom,+ thr_tmem_store: cute.CopyAtom,+ thr_tmem_store_scale: cute.CopyAtom,+ tStS_t2r: cute.Tensor,+ tStScale_r2t: cute.Tensor,+ tStP_r2t: cute.Tensor,+ sScale: cute.Tensor,+ stage: int | Int32,+ batch_idx: Int32,+ head_idx: Int32,+ m_block: Int32,+ seqlen,+ aux_tensors: Optional[list] = None,+ fastdiv_mods=(None, None),+ mask_fn: Optional[Callable] = None,+ is_first: bool = False,+ tCtSFP: Optional[cute.Tensor] = None,+ sSFP: Optional[cute.Tensor] = None,+ ) -> Tuple[cute.Int32, cute.Int32, cute.Int32]:+ """Perform a single step of the softmax computation on a block of attention scores.++ This method processes one block of the attention matrix, computing numerically stable+ softmax by first finding the row maximum, subtracting it from all elements, applying+ exponential function, and then normalizing by the sum of exponentials. It also handles+ optional masking of attention scores.++ The method involves several key operations:+ 1. Loading attention scores from tensor memory+ 2. Applying optional masking based on position+ 3. Computing row-wise maximum values for numerical stability+ 4. Transforming scores using exp2(x*scale - max*scale)+ 5. Computing row sums for normalization+ 6. Coordinating pipeline synchronization between different processing stages+ """+ tilePlikeFP32 = self.mma_tiler_qk[1] // Float32.width * self.v_dtype.width+ tScS = thr_mma_qk.partition_C(cute.make_identity_tensor(self.mma_tiler_qk[:2]))+ tScScale = cute.composition(tScS, cute.make_layout((self.m_block_size, 1)))+ # P size when in FP32+ tScP = cute.composition(tScS, cute.make_layout((self.m_block_size, tilePlikeFP32)))++ # Wait for Si+ iket.range_push("sm_wait_s")+ cute.arch.mbarrier_wait(mbar_ptr + self.mbar_S_full_offset + stage, mma_si_consumer_phase)+ iket.range_pop()+ tSrS_t2r = cute.make_rmem_tensor(thr_tmem_load.partition_D(tScS).shape, self.qk_acc_dtype)+ cute.copy(thr_tmem_load, tStS_t2r, tSrS_t2r)++ # unblock sfqk load+ cute.arch.fence_view_async_tmem_load()+ sfqk_stage = self.q_stage - 1 - stage+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_sfqk_load_offset + sfqk_stage)++ if cutlass.const_expr(self.score_mod is not None):+ self.apply_score_mod(+ tSrS_t2r,+ thr_tmem_load,+ thr_mma_qk,+ batch_idx,+ head_idx,+ m_block,+ n_block,+ softmax,+ aux_tensors,+ fastdiv_mods,+ )++ if const_expr(mask_fn is not None):+ mask_fn(tSrS_t2r, n_block=n_block)++ iket.range_push("sm_rowmax")+ row_max, acc_scale = softmax.update_row_max(tSrS_t2r.load(), is_first)+ iket.range_pop()+ tSrPSF_f32 = None+ tSrPSF = None++ if const_expr(not is_first):+ # tSrScale_r2t = cute.make_rmem_tensor(thr_tmem_store_scale.partition_S(tScScale).shape, Float32)+ # tSrScale_r2t[0] = acc_scale+ # cute.copy(thr_tmem_store_scale, tSrScale_r2t, tStScale_r2t)+ # cute.arch.fence_view_async_tmem_store()+ thread_idx = thr_tmem_load.thr_idx+ sScale[thread_idx + stage * self.m_block_size] = acc_scale+ # Notify correction wg that row_max is ready+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_full_offset + stage)++ softmax.scale_subtract_rowmax(tSrS_t2r, row_max)+ # Sequence barrier wait+ if const_expr(self.s0_s1_barrier):+ cute.arch.mbarrier_wait(+ mbar_ptr + mbar_s0_s1_sequence_offset + stage * 4, s0_s1_sequence_phase+ )+ tSrP_r2t_f32 = cute.make_rmem_tensor(thr_tmem_store.partition_S(tScP).shape, Float32)+ tSrP_r2t = cute.make_tensor(+ cute.recast_ptr(tSrP_r2t_f32.iterator, dtype=self.v_dtype),+ tSrS_t2r.layout, # shape of S owned by this thread+ )+++ if const_expr(self.quant_pv):+ iket.range_push("sm_exp")+ # Exp2 with softmax scale and sp1 scaling+ softmax.apply_exp2_convert(+ tSrS_t2r,+ e2e=mask_fn is None and self.head_dim_padded <= 128,+ e2e_freq=self.e2e_freq,+ )+ # update_row_sum BEFORE scale_groupwise so it uses original P values+ softmax.update_row_sum(tSrS_t2r.load(), acc_scale, is_first)+ iket.range_pop()+ # Everything below is the FP4-only P re-quantization: a segmented+ # group max, a groupwise rescale, the FP4 convert, and the scale+ # factor R2S. The BF16 path reaches the TMEM store directly from+ # apply_exp2_convert, so this range is the extra cost FP4 pays on+ # the softmax critical path.+ iket.range_push("sm_pquant")+ tSrPSF_f32 = softmax.compute_group_max(tSrS_t2r, sf_size=self.sf_vec_size)+ tSrPSF = cute.make_rmem_tensor(tSrPSF_f32.layout, cute.Float8E4M3FN)+ softmax.scale_groupwise(tSrS_t2r, tSrPSF_f32, sf_size=self.sf_vec_size)+ self._quant_fp4(tSrS_t2r, tSrPSF_f32, tSrP_r2t, tSrPSF)+ # R2S: Copy tSrPSF (registers) to sSFP (shared memory).+ # The SFP smem layout is BlockScaledBasicChunk(16) tile_to_shape((M=128, K=128)),+ # giving byte offsets:+ # byte = (m%32)*16 + ((m//32)%4)*4 + (k_block%4) + (k_block//4)*512+ # Each softmax thread holds 1 M row (lane_id within warp = row in [0,32),+ # warp_id within softmax warpgroup = row block in [0,4)) and 8 K groups.+ # The first 4 K groups land at +0,+1,+2,+3 within the row's atom; the+ # next 4 land at +512,+513,+514,+515 (rest_k stride).+ if const_expr(sSFP is not None):+ thread_idx = thr_tmem_load.thr_idx+ lane_id = thread_idx % 32+ warp_id = thread_idx // 32+ base_offset = lane_id * 16 + (warp_id % 4) * 4+ sfp_thread_layout = cute.make_layout((4, 2), stride=(1, 512))+ sSFP_stage_ptr = sSFP[None, None, None, stage].iterator+ sSFP_thread = cute.make_tensor(sSFP_stage_ptr + base_offset, sfp_thread_layout)+ tSrPSF_2d = cute.logical_divide(tSrPSF, cute.make_layout(4))+ cute.autovec_copy(tSrPSF_2d, sSFP_thread)+ # These stores go through the generic proxy, but the MMA warp+ # reads sSFP with tcgen05.cp, an async-proxy access. The+ # mbarrier arrive below orders generic-proxy traffic only, so+ # without this fence the S2T copy can still observe the+ # previous n_block's scale factors.+ cute.arch.fence_view_async_shared()+ iket.range_pop()+ else:+ # softmax.scale_apply_exp2_convert(tSrS_t2r, row_max, tSrP_r2t)+ softmax.apply_exp2_convert(+ tSrS_t2r,+ tSrP_r2t,+ e2e=mask_fn is None and self.head_dim_padded <= 128,+ e2e_freq=self.e2e_freq,+ )+ # Sequence barrier arrive+ if const_expr(self.s0_s1_barrier):+ cute.arch.mbarrier_arrive(mbar_ptr + mbar_s0_s1_sequence_offset + (1 - stage) * 4)+ for i in cutlass.range_constexpr(self.mbar_p_split(cute.size(tStP_r2t.shape[2]))):+ cute.copy(thr_tmem_store, tSrP_r2t_f32[None, None, i], tStP_r2t[None, None, i])+ cute.arch.fence_view_async_tmem_store()+ # Notify mma warp that P is ready (and SFP is in SMEM)+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage)+ for i in cutlass.range_constexpr(self.mbar_p_split(cute.size(tStP_r2t.shape[2])), cute.size(tStP_r2t.shape[2])):+ cute.copy(thr_tmem_store, tSrP_r2t_f32[None, None, i], tStP_r2t[None, None, i])+ cute.arch.fence_view_async_tmem_store()+ # Notify mma warp that the 2nd half of P is ready+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_P_full_2_offset + stage)+ iket.range_push("sm_wait_corr")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_softmax_corr_empty_offset + stage, si_corr_producer_phase+ )+ iket.range_pop()++ if const_expr(not self.quant_pv):+ softmax.update_row_sum(tSrS_t2r.load(), acc_scale, is_first)+ # acc_scale = cute.arch.exp2(acc_scale_)+ return mma_si_consumer_phase ^ 1, si_corr_producer_phase ^ 1, s0_s1_sequence_phase ^ 1++ @cute.jit+ def softmax_residual_step(+ self,+ mma_si_consumer_phase: Int32,+ si_corr_producer_phase: Int32,+ s0_s1_sequence_phase: Int32,+ softmax: SoftmaxSm100,+ mbar_ptr: cute.Pointer,+ thr_mma_qk: cute.ThrMma,+ thr_tmem_load: cute.CopyAtom,+ thr_tmem_store_bf16: cute.CopyAtom,+ thr_tmem_store_scale: cute.CopyAtom,+ tStS_t2r: cute.Tensor,+ tStP_bf16_r2t: cute.Tensor,+ sScale: cute.Tensor,+ stage: int | Int32,+ batch_idx: Int32,+ m_block: Int32,+ mResidualSeqUsedK: cute.Tensor,+ thread_idx: Int32,+ split_idx: Int32 = Int32(0),+ ) -> Tuple[cute.Int32, cute.Int32, cute.Int32]:+ """Compute the BF16 residual softmax step.++ BF16 variant of softmax_step for the residual block. Reads BF16-derived+ FP32 scores from TMEM (written by BF16 GEMM_QK in MMA warp), applies+ mResidualSeqUsedK mask, computes row_max, exp2, row_sum (is_first=True+ so initializes the softmax state from this block alone), casts P→BF16,+ stores P_BF16 to TMEM at tmem_p_bf16_offset (separate from FP4 P).++ After this step, softmax.row_max / softmax.row_sum hold the BF16-block+ contribution. The FP4 mainloop's first softmax_step then runs with+ is_first=False so the online-softmax merge folds in the FP4 blocks on+ top of the BF16 seed.+ """+ tilePlikeFP32_bf16 = self.mma_tiler_qk[1] // Float32.width * 16 # BF16=16 bits+ tScS = thr_mma_qk.partition_C(cute.make_identity_tensor(self.mma_tiler_qk[:2]))+ tScP_bf16 = cute.composition(tScS, cute.make_layout((self.m_block_size, tilePlikeFP32_bf16)))++ iket.range_push("sm_res_wait_s")+ cute.arch.mbarrier_wait(mbar_ptr + self.mbar_bf16_S_full_offset, Int32(0))+ iket.range_pop()+ iket.range_push("sm_res_comp")+ tSrS_t2r = cute.make_rmem_tensor(thr_tmem_load.partition_D(tScS).shape, self.qk_acc_dtype)+ cute.copy(thr_tmem_load, tStS_t2r, tSrS_t2r)+ cute.arch.fence_view_async_tmem_load()++ # Apply mResidualSeqUsedK mask: tokens at j >= seqused_k[batch_idx] → -inf.+ # Values are in [0, BLOCK_SIZE]. A zero-length row still participates in+ # the residual pipeline barriers, but below it writes P=0 and leaves the+ # reset online-softmax state (-inf max, zero sum) untouched. The first+ # FP4 tile can therefore merge with is_first=False without NaNs.+ seqused_k = Int32(self.n_block_size)+ if const_expr(mResidualSeqUsedK is not None):+ seqused_k = mResidualSeqUsedK[batch_idx]+ if const_expr(self.is_split_kv and not self.bf16_split):+ # One block, one owner: the other splits run the same pipeline+ # over a length of zero and contribute nothing, so the combine+ # sees the residual exactly once.+ if split_idx != self.residual_split_idx:+ seqused_k = Int32(0)+ tScS_t2r = thr_tmem_load.partition_D(tScS)+ for i in cutlass.range_constexpr(cute.size(tSrS_t2r)):+ n_coord = tScS_t2r[i][1] # N coordinate+ if n_coord >= seqused_k:+ tSrS_t2r[i] = -Float32.inf++ # Build BF16 P fragment in registers (cast destination).+ tSrP_bf16_r2t_view = cute.make_rmem_tensor(+ thr_tmem_store_bf16.partition_S(tScP_bf16).shape, Float32+ )+ # The BF16 P fragment is the same logical shape as tSrS_t2r but typed+ # as BFloat16. We use the f32 register and cast inside apply_exp2_convert.+ tSrP_bf16 = cute.make_tensor(+ cute.recast_ptr(tSrP_bf16_r2t_view.iterator, dtype=cutlass.BFloat16),+ tSrS_t2r.layout,+ )++ if seqused_k > Int32(0):+ # Compute row_max with is_first=True (initializes softmax state).+ row_max, acc_scale = softmax.update_row_max(+ tSrS_t2r.load(), is_first=True+ )+ # In-place compute (S - row_max) * scale_log2.+ softmax.scale_subtract_rowmax(tSrS_t2r, row_max)+ # exp2 in FP32 + cast to BF16 destination.+ softmax.apply_exp2_convert(+ tSrS_t2r,+ tSrP_bf16,+ e2e=False,+ )+ softmax.update_row_sum(+ tSrS_t2r.load(), acc_scale, is_first=True+ )+ else:+ # The MMA/PV pipeline must still observe its normal producer+ # arrivals, so publish an all-zero P tile instead of skipping it.+ tSrP_bf16_r2t_view.fill(Float32(0.0))++ # Store BF16 P to TMEM (split into two halves to overlap with MMA).+ for i in cutlass.range_constexpr(self.mbar_p_split(cute.size(tStP_bf16_r2t.shape[2]))):+ cute.copy(thr_tmem_store_bf16, tSrP_bf16_r2t_view[None, None, i], tStP_bf16_r2t[None, None, i])+ cute.arch.fence_view_async_tmem_store()+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_bf16_P_full_offset)+ for i in cutlass.range_constexpr(+ self.mbar_p_split(cute.size(tStP_bf16_r2t.shape[2])),+ cute.size(tStP_bf16_r2t.shape[2]),+ ):+ cute.copy(thr_tmem_store_bf16, tSrP_bf16_r2t_view[None, None, i], tStP_bf16_r2t[None, None, i])+ cute.arch.fence_view_async_tmem_store()+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_bf16_P_full_2_offset)+ iket.range_pop()+++ # Phase tracking: mma_si_consumer_phase is unchanged because we+ # didn't consume mbar_S_full; si_corr_producer_phase is unchanged+ # because we didn't consume mbar_softmax_corr_empty.+ return mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase++ @cute.jit+ def correction_loop(+ self,+ thr_mma_qk: cute.ThrMma,+ thr_mma_pv: cute.ThrMma,+ tStS: cute.Tensor,+ tOtOs: tuple[cute.Tensor],+ sScale: cute.Tensor,+ mO: cute.Tensor,+ mLSE: cute.Tensor,+ sO: cute.Tensor,+ learnable_sink: Optional[cute.Tensor],+ gmem_tiled_copy_O: cute.TiledCopy,+ tma_atom_O: cute.CopyAtom,+ mbar_ptr: cute.Pointer,+ softmax_scale_log2: Float32,+ block_info: BlockInfo,+ num_splits: Int32,+ SeqlenInfoCls: Callable,+ TileSchedulerCls: Callable,+ blocksparse_tensors: Optional[BlockSparseTensors] = None,+ role: cutlass.Constexpr = SplitRole.ALL,+ ):+ runs_pages = const_expr(role != SplitRole.BF16)+ runs_tail = const_expr(+ self.fused_residual_first_block and role != SplitRole.PAGES+ )+ runs_sink = const_expr(self.fused_sink_block and role != SplitRole.PAGES)+ two_level = const_expr(self.p_scale_two_level and runs_pages)+ accumulated = None+ previous_outer = Float32(1.0)+ if const_expr(two_level):+ assert self.q_stage == 1+ accumulated = cute.make_rmem_tensor(cute.make_layout(self.head_dim_v_padded), Float32)+ tidx = cute.arch.thread_idx()[0] % (cute.arch.WARP_SIZE * len(self.correction_warp_ids))+ tScS = thr_mma_qk.partition_C(cute.make_identity_tensor(self.mma_tiler_qk[:2]))+ # O^T is a 128-lane FP32 accumulator like S, so S's tensor-memory+ # layout at O's columns addresses it; column r is query row r.+ tOtOs_corr = tOtOs+ if const_expr(self.narrow_pv):+ tOtOs_corr = (+ cute.make_tensor(tStS.iterator + self.tmem_o_offset[0], tStS.layout),+ )+ tStScale_layout = cute.composition(tStS.layout, cute.make_layout((self.m_block_size, 1)))+ tStScales = tuple(+ cute.make_tensor(tStS.iterator + self.tmem_vec_offset[stage], tStScale_layout)+ for stage in range(self.q_stage)+ )+ tScScale = cute.composition(tScS, cute.make_layout((self.m_block_size, 1)))+ tmem_load_v_atom = cute.make_copy_atom(+ tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(1)),+ self.qk_acc_dtype,+ )+ thr_tmem_load_vec = tcgen05.make_tmem_copy(tmem_load_v_atom, tStScales[0]).get_slice(tidx)++ tStScales_t2r = [thr_tmem_load_vec.partition_S(tStScales[stage]) for stage in range(self.q_stage)]+ tSrScale_t2r_shape = thr_tmem_load_vec.partition_D(tScScale).shape++ # The free arrival that lets the mma warp past its first PV. Only a tile+ # starting on an fp4 block gets it; see correction_consume_scale.+ if const_expr(not runs_tail):+ for stage_init in cutlass.range_constexpr(self.q_stage):+ cute.arch.mbarrier_arrive(+ mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage_init+ )++ softmax_corr_consumer_phase = Int32(0)+ o_corr_consumer_phase = Int32(0)+ pv_consumer_phase = Int32(0)+ corr_epi_producer_phase = Int32(1)++ tile_scheduler = TileSchedulerCls()+ work_tile = tile_scheduler.initial_work_tile_info()+ while work_tile.is_valid_tile:+ m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx+ seqlen = SeqlenInfoCls(batch_idx)+ n_block_min, n_block_max = self.split_block_range(+ block_info, seqlen, m_block, split_idx, num_splits+ )++ if const_expr(self.is_split_kv):+ mO_cur = seqlen.offset_batch_Q(mO, batch_idx, dim=3)[None, None, head_idx, split_idx]+ else:+ mO_cur = seqlen.offset_batch_Q(mO, batch_idx, dim=3)[None, None, head_idx]+ gO = cute.local_tile(mO_cur, (self.m_block_size, self.head_dim_v_padded), (None, 0))++ # Default LSE to -inf for invalid split_idx tiles+ stats = [(0.0, -Float32.inf if const_expr(mLSE is not None or learnable_sink is not None) else None, True)] * self.q_stage++ if const_expr(self.use_block_sparsity):+ total_block_count = get_total_block_count(blocksparse_tensors, batch_idx, head_idx, m_block)+ has_work = total_block_count > Int32(0)+ else:+ total_block_count = n_block_max - n_block_min+ has_work = self.tile_has_work(+ n_block_min, n_block_max, split_idx, num_splits+ )++ if has_work:+ if const_expr(two_level):+ accumulated.fill(0.0)+ previous_outer = Float32(1.0)+ if const_expr(runs_pages):+ # The first signal from softmax needs no correction unless a+ # residual seeded the online softmax, in which case it carries+ # the factor that takes O from the residual's row max to the+ # first fp4 block's.+ iket.range_push("corr_wait_sm")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_softmax_corr_full_offset + 0, softmax_corr_consumer_phase+ )+ if const_expr(runs_tail):+ previous_outer = self.correction_consume_scale(+ thr_mma_pv, tOtOs_corr, sScale, tidx, 0, mbar_ptr,+ accumulated=accumulated, previous_outer=previous_outer,+ tScS=tScS,+ )+ elif const_expr(two_level):+ previous_outer = sScale[tidx + self.m_block_size * 2] if tidx < self.transposed_query_rows else Float32(0.0)+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_empty_offset + 0)+ if const_expr(self.q_stage == 2):+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_softmax_corr_full_offset + 1, softmax_corr_consumer_phase+ )+ if const_expr(runs_tail):+ self.correction_consume_scale(+ thr_mma_pv, tOtOs_corr, sScale, tidx, 1, mbar_ptr,+ tScS=tScS,+ )+ iket.range_pop()+ softmax_corr_consumer_phase ^= 1++ tSrScale_t2r = cute.make_rmem_tensor(tSrScale_t2r_shape, Float32)+ # The sink's factor comes over the same handshake as a+ # page's, so it takes one more step of this loop. A second+ # copy of the rescale after the loop, never mind whether it+ # runs, compiles every block of the loop longer: 2.5%+ # slower at batch 32 on Qwen3-4B. A tile whose fp4 length is+ # zero still runs its first step.+ correction_steps = total_block_count - 1+ if const_expr(runs_sink):+ correction_steps = cutlass.max(correction_steps, Int32(0)) + 1+ for i in cutlass.range(correction_steps, unroll=1):+ for stage in cutlass.range_constexpr(self.q_stage):+ # wait for S0 / S1+ iket.range_push("corr_wait_sm")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_softmax_corr_full_offset + stage,+ softmax_corr_consumer_phase,+ )+ iket.range_pop()+ previous_outer = self.correction_consume_scale(+ thr_mma_pv, tOtOs_corr, sScale, tidx, stage, mbar_ptr,+ previous_pv_phase=pv_consumer_phase ^ (i & 1),+ accumulated=accumulated, previous_outer=previous_outer,+ tScS=tScS,+ )+ if const_expr(self.q_stage == 2):+ cute.arch.mbarrier_arrive(+ mbar_ptr + self.mbar_softmax_corr_empty_offset + (1 - stage)+ )+ else:+ cute.arch.mbarrier_arrive(+ mbar_ptr + self.mbar_softmax_corr_empty_offset + stage+ )+ softmax_corr_consumer_phase ^= 1+ if const_expr(self.transpose_s):+ # P has one completion per FP4 page, including the+ # zero-page placeholder. The correction handshake also+ # signals the final row sum, so its phase cannot track+ # P across tiles of a persistent CTA.+ pv_consumer_phase ^= cutlass.max(total_block_count, Int32(1)) & 1+ if const_expr(runs_sink and not runs_pages):+ iket.range_push("corr_wait_sm")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_softmax_corr_full_offset,+ softmax_corr_consumer_phase,+ )+ iket.range_pop()+ self.correction_consume_scale(+ thr_mma_pv, tOtOs_corr, sScale, tidx, 0, mbar_ptr,+ tScS=tScS,+ )+ cute.arch.mbarrier_arrive(+ mbar_ptr + self.mbar_softmax_corr_empty_offset+ )+ softmax_corr_consumer_phase ^= 1+ if const_expr(self.q_stage == 2):+ cute.arch.mbarrier_arrive(+ mbar_ptr + self.mbar_softmax_corr_empty_offset + 1+ )+ # End of seqlen_corr_loop_steps++ # Even in the case of self.overlap_sO_sQ, we can write to stage 0 of sO without+ # additional sync because the MMA in the top half must have been done.+ # Similarly we can write to stage 1 of sO without additional sync.+ # Over the BF16 tiles the O_full wait below suffices: it follows+ # the tile's last MMA, and that MMA is their last reader.+ learnable_sink_val = [None] * self.q_stage+ if const_expr(learnable_sink is not None):+ if const_expr(not self.pack_gqa):+ sink_val = Float32(learnable_sink[head_idx])+ learnable_sink_val = [sink_val] * self.q_stage+ else: # Each thread might have a different sink value due to different q_head+ for stage in cutlass.range_constexpr(self.q_stage):+ q_head_idx = (+ (self.q_stage * m_block + stage) * self.m_block_size + tidx+ ) % self.qhead_per_kvhead + head_idx * self.qhead_per_kvhead+ learnable_sink_val[stage] = Float32(learnable_sink[q_head_idx])+ for stage in cutlass.range_constexpr(self.q_stage):+ iket.range_push("corr_wait_sm")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_softmax_corr_full_offset + stage,+ softmax_corr_consumer_phase,+ )+ iket.range_pop()+ # cute.copy(tiled_tmem_load_vec, tStScales_t2r[stage], tSrScale_t2r)+ # cute.arch.fence_view_async_tmem_load()+ # scale = tSrScale_t2r[0]+ if const_expr(self.transpose_s):+ # A zero sum marks the row as empty further down, which+ # is what the rows past the live query rows are.+ row_sum = (+ sScale[tidx + stage * self.m_block_size]+ if tidx < self.transposed_query_rows+ else Float32(0.0)+ )+ else:+ row_sum = sScale[tidx + stage * self.m_block_size]+ if const_expr(mLSE is not None or learnable_sink is not None):+ row_max = sScale[tidx + stage * self.m_block_size + self.m_block_size * 2]+ if const_expr(self.transpose_s):+ row_max = (+ row_max+ if tidx < self.transposed_query_rows+ else -Float32.inf+ )+ else:+ row_max = None+ # A narrow PV leaves every live row's lane of O^T in every+ # thread, so each one needs all the row sums, and has to+ # read them before softmax is free to overwrite sScale.+ row_scales = None+ if const_expr(self.narrow_pv):+ row_scales = cute.make_rmem_tensor(+ cute.make_layout(self.transposed_query_rows), Float32+ )+ for row in cutlass.range_constexpr(self.transposed_query_rows):+ live_sum = sScale[row + stage * self.m_block_size]+ row_scales[row] = cute.arch.rcp_approx(+ live_sum+ if not (live_sum == 0.0 or live_sum != live_sum)+ else 1.0+ )+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_empty_offset + stage)+ if const_expr(learnable_sink is not None):+ LOG2_E = math.log2(math.e)+ sink_val = learnable_sink_val[stage]+ if const_expr(not self.is_split_kv) or split_idx == 0:+ if row_max == -Float32.inf:+ # It's possible to have an empty row with splitKV.+ row_max = sink_val * (LOG2_E / softmax_scale_log2)+ row_sum = Float32(2.0 ** self.p_scale_rescale_log2)+ else:+ row_sum += utils.exp2f(+ sink_val * LOG2_E - row_max * softmax_scale_log2+ + Float32(self.p_scale_rescale_log2)+ )+ acc_O_mn_row_is_zero_or_nan = row_sum == 0.0 or row_sum != row_sum+ stats[stage] = (row_sum, row_max, acc_O_mn_row_is_zero_or_nan)+ scale = cute.arch.rcp_approx(row_sum if not acc_O_mn_row_is_zero_or_nan else 1.0)+ iket.range_push("corr_wait_o")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_O_full_offset + stage, o_corr_consumer_phase+ )+ iket.range_pop()+ if const_expr(not self.use_correction_warps_for_epi):+ iket.range_push("corr_wait_epi")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_corr_epi_empty_offset + stage, corr_epi_producer_phase+ )+ iket.range_pop()+ if const_expr(two_level):+ self.correction_accumulate_tile(+ thr_mma_pv, tOtOs[stage], tidx, accumulated,+ previous_outer, Float32(1.0), store=True,+ )+ if const_expr(self.narrow_pv):+ self.correction_epilogue_rows(+ tOtOs_corr[0], tScS, tidx, row_scales, sO[None, None, stage]+ )+ else:+ self.correction_epilogue(+ thr_mma_pv,+ tOtOs[stage],+ tidx,+ stage,+ m_block,+ seqlen.seqlen_q,+ scale,+ sO[None, None, stage],+ mO_cur,+ gO,+ gmem_tiled_copy_O,+ )+ if const_expr(not self.use_correction_warps_for_epi):+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_corr_epi_full_offset + stage)+ # Signal for the next work tile that O buffers in tmem are already read, so+ # mma warp can write to them. Withheld on a residual-seeded+ # tile for the same reason as the pre-loop arrival; see+ # correction_consume_scale.+ if const_expr(not runs_tail):+ cute.arch.mbarrier_arrive(+ mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage+ )++ o_corr_consumer_phase ^= 1+ softmax_corr_consumer_phase ^= 1+ corr_epi_producer_phase ^= 1+ else:+ # WARNING: we need some code before the const_expr, see https://github.com/NVIDIA/cutlass/issues/2781+ if const_expr(self.use_correction_warps_for_epi):+ gmem_tiled_copy_O_for_empty_tile = gmem_tiled_copy_O+ else:+ gmem_tiled_copy_O_for_empty_tile = None+ if const_expr(self.use_block_sparsity):+ (+ softmax_corr_consumer_phase,+ o_corr_consumer_phase,+ corr_epi_producer_phase,+ ) = handle_block_sparse_empty_tile_correction_sm100(+ tidx,+ self.q_stage,+ self.m_block_size,+ self.qhead_per_kvhead,+ self.pack_gqa,+ self.is_split_kv,+ learnable_sink,+ mLSE,+ seqlen,+ m_block,+ head_idx,+ batch_idx,+ split_idx,+ sScale,+ stats,+ self.correction_epilogue,+ thr_mma_pv,+ tOtOs,+ sO,+ mbar_ptr,+ self.mbar_softmax_corr_full_offset,+ self.mbar_softmax_corr_empty_offset,+ self.mbar_P_full_O_rescaled_offset,+ self.mbar_P_full_2_offset,+ self.mbar_corr_epi_full_offset,+ self.mbar_corr_epi_empty_offset,+ softmax_corr_consumer_phase,+ o_corr_consumer_phase,+ corr_epi_producer_phase,+ softmax_scale_log2,+ mO_cur,+ gO,+ gmem_tiled_copy_O_for_empty_tile,+ )++ if const_expr(mLSE is not None):+ if const_expr(not seqlen.has_cu_seqlens_q):+ if const_expr(self.is_split_kv):+ mLSE_cur = mLSE[None, head_idx, batch_idx, split_idx]+ else:+ mLSE_cur = mLSE[None, head_idx, batch_idx]+ else:+ offset = (+ seqlen.offset_q if const_expr(not self.pack_gqa) else (0, seqlen.offset_q)+ )+ if const_expr(self.is_split_kv):+ mLSE_cur = cute.domain_offset((offset,), mLSE[None, head_idx, split_idx])+ else:+ mLSE_cur = cute.domain_offset((offset,), mLSE[None, head_idx])+ for stage in cutlass.range_constexpr(self.q_stage):+ gLSE = cute.local_tile(+ mLSE_cur, (self.m_block_size,), (self.q_stage * m_block + stage,)+ )+ row_sum, row_max, acc_O_mn_row_is_zero_or_nan = stats[stage]+ LN2 = math.log(2.0)+ lse = (+ (row_max * softmax_scale_log2 + utils.log2f(row_sum)) * LN2+ if not acc_O_mn_row_is_zero_or_nan+ else -Float32.inf+ )+ if const_expr(self.p_scale_rescale_log2):+ lse -= Float32(self.p_scale_rescale_log2 * LN2)+ effective_seqlen_q_lse = (+ Int32(1)+ if const_expr(self.seqlen_q_static_one)+ else seqlen.seqlen_q+ )+ seqlen_q = (+ effective_seqlen_q_lse+ if const_expr(not self.pack_gqa)+ else effective_seqlen_q_lse * self.qhead_per_kvhead+ )+ if tidx < seqlen_q - (self.q_stage * m_block + stage) * self.m_block_size:+ # This actually just works with PackGQA too+ gLSE[tidx] = lse++ # Advance to next tile+ tile_scheduler.advance_to_next_work()+ work_tile = tile_scheduler.get_current_work()+ # End of persistent scheduler loop++ @cute.jit+ def correction_rescale(+ self,+ thr_mma: cute.ThrMma,+ tOtO: cute.Tensor,+ tidx: Int32,+ scale: Float32,+ ):+ """Rescale intermediate attention results based on softmax normalization factor.++ This method performs a crucial correction step in the attention computation pipeline.+ When processing attention in blocks, the softmax normalization factors may change+ as new blocks are processed. This method rescales previously computed partial+ output values to account for updated normalization factors.++ The implementation uses efficient tensor memory operations to:+ 1. Load existing partial attention output from tensor memory+ 2. Apply the scaling factor to all elements+ 3. Store the rescaled results back to tensor memory+ """+ tOcO = thr_mma.partition_C(cute.make_identity_tensor(self.mma_tiler_pv[:2]))+ corr_tile_size = 16 # tuneable parameter+ tmem_load_atom = cute.make_copy_atom(+ tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(corr_tile_size)),+ self.pv_acc_dtype,+ )+ tmem_store_atom = cute.make_copy_atom(+ tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(corr_tile_size)),+ self.pv_acc_dtype,+ )+ tOtO_i = cute.composition(tOtO, cute.make_layout((self.m_block_size, corr_tile_size)))+ tOcO_i = cute.composition(tOcO, cute.make_layout((self.m_block_size, corr_tile_size)))+ thr_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tOtO_i).get_slice(tidx)+ thr_tmem_store = tcgen05.make_tmem_copy(tmem_store_atom, tOtO_i).get_slice(tidx)+ tOtO_t2r = thr_tmem_load.partition_S(tOtO_i)+ tOrO_t2r_shape = thr_tmem_load.partition_D(tOcO_i).shape+ tOtO_r2t = thr_tmem_store.partition_D(tOtO_i)+ frg_count = self.head_dim_v_padded // corr_tile_size+ tOrO_frg = cute.make_rmem_tensor((tOrO_t2r_shape, frg_count), self.pv_acc_dtype)+ for i in cutlass.range_constexpr(frg_count):+ tOrO_frg = cute.make_rmem_tensor(tOrO_t2r_shape, self.pv_acc_dtype)+ tOtO_t2r_i = cute.make_tensor(tOtO_t2r.iterator + i * corr_tile_size, tOtO_t2r.layout)+ cute.copy(thr_tmem_load, tOtO_t2r_i, tOrO_frg)+ for j in cutlass.range(0, cute.size(tOrO_frg), 2, unroll_full=True):+ tOrO_frg[j], tOrO_frg[j + 1] = utils.mul_packed_f32x2(+ (tOrO_frg[j], tOrO_frg[j + 1]),+ (scale, scale),+ )+ tOtO_r2t_i = cute.make_tensor(tOtO_r2t.iterator + i * corr_tile_size, tOtO_r2t.layout)+ cute.copy(thr_tmem_store, tOrO_frg, tOtO_r2t_i)+ cute.arch.fence_view_async_tmem_store()++ def _o_transposed_copies(self, tOtT: cute.Tensor, tScS: cute.Tensor, tidx: Int32):+ """Tensor-memory copies over eight rows of O^T at a time.++ A thread gets one head-dimension lane and eight consecutive columns,+ which are query rows; fragment element j is column j.+ """+ chunk = 8+ tile = cute.composition(tOtT, cute.make_layout((self.m_block_size, chunk)))+ coords = cute.composition(tScS, cute.make_layout((self.m_block_size, chunk)))+ load = tcgen05.make_tmem_copy(+ cute.make_copy_atom(+ tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(chunk)), self.pv_acc_dtype+ ),+ tile,+ ).get_slice(tidx)+ store = tcgen05.make_tmem_copy(+ cute.make_copy_atom(+ tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(chunk)), self.pv_acc_dtype+ ),+ tile,+ ).get_slice(tidx)+ return (+ chunk,+ load,+ store,+ load.partition_S(tile),+ store.partition_D(tile),+ load.partition_D(coords).shape,+ )++ @cute.jit+ def correction_rescale_rows(+ self,+ tOtT: cute.Tensor,+ tScS: cute.Tensor,+ tidx: Int32,+ sScale: cute.Tensor,+ stage: int,+ ):+ """Rescale O^T by each live row's factor; see ``narrow_pv``.++ The columns past the live rows hold the product of P's zero rows and+ are left alone.+ """+ chunk, load, store, source, destination, frg_shape = self._o_transposed_copies(+ tOtT, tScS, tidx+ )+ for c in cutlass.range_constexpr(self.pv_rows // chunk):+ fragment = cute.make_rmem_tensor(frg_shape, self.pv_acc_dtype)+ cute.copy(load, cute.make_tensor(source.iterator + c * chunk, source.layout), fragment)+ for j in cutlass.range_constexpr(chunk):+ if const_expr(c * chunk + j < self.transposed_query_rows):+ fragment[j] = fragment[j] * sScale[c * chunk + j + stage * self.m_block_size]+ cute.copy(+ store, fragment, cute.make_tensor(destination.iterator + c * chunk, destination.layout)+ )+ cute.arch.fence_view_async_tmem_store()++ @cute.jit+ def correction_epilogue_rows(+ self,+ tOtT: cute.Tensor,+ tScS: cute.Tensor,+ tidx: Int32,+ row_scales: cute.Tensor,+ sO: cute.Tensor,+ ):+ """Normalize O^T and store it into the row-major O tile.++ Each thread writes its head-dimension lane of every live row, so the+ epilogue warp copies sO out exactly as it does the wide tile.+ """+ chunk, load, _, source, _, frg_shape = self._o_transposed_copies(tOtT, tScS, tidx)+ for c in cutlass.range_constexpr(self.pv_rows // chunk):+ fragment = cute.make_rmem_tensor(frg_shape, self.pv_acc_dtype)+ cute.copy(load, cute.make_tensor(source.iterator + c * chunk, source.layout), fragment)+ for j in cutlass.range_constexpr(chunk):+ if const_expr(c * chunk + j < self.transposed_query_rows):+ fragment[j] = fragment[j] * row_scales[c * chunk + j]+ converted = cute.make_rmem_tensor(frg_shape, self.o_dtype)+ converted.store(fragment.load().to(self.o_dtype))+ for j in cutlass.range_constexpr(chunk):+ if const_expr(c * chunk + j < self.transposed_query_rows):+ sO[c * chunk + j, tidx] = converted[j]+ cute.arch.fence_view_async_tmem_load()+ cute.arch.fence_proxy("async.shared", space="cta")++ @cute.jit+ def correction_accumulate_tile(+ self, thr_mma: cute.ThrMma, tOtO: cute.Tensor, tidx: Int32,+ accumulated: cute.Tensor, outer: Float32, rescale: Float32,+ store: cutlass.Constexpr[bool] = False,+ ):+ """Merge one PV tile after undoing its FP32 outer scale exactly once.++ The running numerator remains in the ordinary online-softmax basis.+ Each FP4 MMA zero-initializes TMEM; the correction handshake prevents+ its reuse until this load completes. BF16 tail and sink use outer=1.+ """+ width = const_expr(16)+ identity = thr_mma.partition_C(cute.make_identity_tensor(self.mma_tiler_pv[:2]))+ tile = cute.composition(tOtO, cute.make_layout((self.m_block_size, width)))+ coordinates = cute.composition(identity, cute.make_layout((self.m_block_size, width)))+ loader = tcgen05.make_tmem_copy(cute.make_copy_atom(+ tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(width)), Float32,+ ), tile).get_slice(tidx)+ source = loader.partition_S(tile)+ fragment = cute.make_rmem_tensor(loader.partition_D(coordinates).shape, Float32)+ writer = tcgen05.make_tmem_copy(cute.make_copy_atom(+ tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(width)), Float32,+ ), tile).get_slice(tidx)+ destination = writer.partition_D(tile)+ for i in cutlass.range_constexpr(self.head_dim_v_padded // width):+ src = cute.make_tensor(source.iterator + i * width, source.layout)+ cute.copy(loader, src, fragment)+ for j in cutlass.range_constexpr(cute.size(fragment)):+ contribution = Float32(0.0)+ if outer > 0.0:+ contribution = fragment[j] * outer+ accumulated[i * width + j] = (accumulated[i * width + j] + contribution) * rescale+ if const_expr(store):+ fragment[j] = accumulated[i * width + j]+ if const_expr(store):+ dst = cute.make_tensor(destination.iterator + i * width, destination.layout)+ cute.copy(writer, fragment, dst)+ cute.arch.fence_view_async_tmem_load()+ if const_expr(store):+ cute.arch.fence_view_async_tmem_store()++ @cute.jit+ def correction_consume_scale(+ self,+ thr_mma_pv: cute.ThrMma,+ tOtOs: tuple[cute.Tensor],+ sScale: cute.Tensor,+ tidx: Int32,+ stage: int,+ mbar_ptr: cute.Pointer,+ previous_pv_phase: Optional[Int32] = None,+ accumulated: Optional[cute.Tensor] = None,+ previous_outer: Float32 = Float32(1.0),+ tScS: Optional[cute.Tensor] = None,+ ):+ """Take one correction factor from softmax and release the mma warp.++ Transposed, softmax publishes one factor per live query row rather than+ one per thread, so the O rows past that bound have no factor and no+ consumer.++ Every step of the loop runs this. The first step of a tile does too,+ but only when a fused residual seeded the online softmax: the+ residual's PV has already written O by then, which makes that first+ factor a real one rather than the no-op it is on a tile whose first+ block zero-initializes O. The arrival matters as much as the rescale —+ it is what holds the mma warp's first fp4 PV until O has been scaled,+ and a residual-seeded tile has no free pre-loop arrival standing in for+ it.++ With a narrow PV ``tOtOs`` holds the O^T view instead, every thread+ holds a lane of every live row, and all of them read every factor.+ """+ if const_expr(self.narrow_pv):+ min_scale = sScale[stage * self.m_block_size]+ for row in cutlass.range_constexpr(1, self.transposed_query_rows):+ min_scale = cutlass.min(min_scale, sScale[row + stage * self.m_block_size])+ if min_scale < 1.0:+ if const_expr(previous_pv_phase is not None):+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_P_consumed_offset + stage,+ previous_pv_phase,+ )+ self.correction_rescale_rows(tOtOs[0], tScS, tidx, sScale, stage)+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage)+ return Float32(1.0)+ if const_expr(self.transpose_s):+ scale = (+ sScale[tidx + stage * self.m_block_size]+ if tidx < self.transposed_query_rows+ else Float32(1.0)+ )+ else:+ scale = sScale[tidx + stage * self.m_block_size]+ next_outer = Float32(1.0)+ if const_expr(accumulated is not None):+ if const_expr(previous_pv_phase is not None):+ cute.arch.mbarrier_wait(mbar_ptr + self.mbar_P_consumed_offset + stage, previous_pv_phase)+ self.correction_accumulate_tile(+ thr_mma_pv, tOtOs[0], tidx, accumulated, previous_outer, scale,+ )+ next_outer = sScale[tidx + self.m_block_size * 2] if tidx < self.transposed_query_rows else Float32(0.0)+ should_rescale = cute.arch.vote_ballot_sync(scale < 1.0) != 0+ if const_expr(accumulated is None) and should_rescale:+ # Transposed QK can finish before the preceding PV is issued.+ # Its softmax signal therefore does not make O safe to rescale.+ # Wait only when touching O; a unit factor needs no PV dependency.+ if const_expr(self.transpose_s and previous_pv_phase is not None):+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_P_consumed_offset + stage,+ previous_pv_phase,+ )+ self.correction_rescale(+ thr_mma_pv, tOtOs[stage if self.q_stage == 2 else 0], tidx, scale+ )+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage)+ return next_outer++ @cute.jit+ def correction_epilogue(+ self,+ thr_mma: cute.ThrMma,+ tOtO: cute.Tensor, # tmem+ tidx: Int32,+ stage: Int32,+ m_block: Int32,+ seqlen_q: Int32,+ scale: Float32,+ sO: cute.Tensor,+ mO_cur: Optional[cute.Tensor] = None,+ gO: Optional[cute.Tensor] = None,+ gmem_tiled_copy_O: Optional[cute.TiledCopy] = None,+ ):+ """Apply final scaling and transformation to attention output before writing to global memory.++ This correction_epilogue function handles the final processing step for attention output values.+ It applies a scaling factor to the accumulated attention results and prepares the+ data for efficient transfer back to global memory.++ The method performs:+ 1. Loading of accumulated attention results from tensor memory+ 2. Application of the final output scaling factor+ 3. Type conversion if necessary (typically from higher precision accumulator to output precision)+ 4. Reorganization of data for optimal memory access patterns+ 5. Preparation for efficient TMA store operations++ :param thr_mma: Thread MMA operation for the computation+ :type thr_mma: cute.ThrMma+ :param tOtO: Tensor containing accumulated attention output+ :type tOtO: cute.Tensor+ :param scale: Final scaling factor(softmax denominator) to apply to the output+ :type scale: Float32+ :param sO: Shared memory tensor for the final output+ :type sO: cute.Tensor+ """++ corr_tile_size = 32 * 8 // self.o_dtype.width+ tOsO = thr_mma.partition_C(sO)+ tOcO = thr_mma.partition_C(cute.make_identity_tensor(self.mma_tiler_pv[:2]))++ tOtO_i = cute.logical_divide(tOtO, cute.make_layout((self.m_block_size, corr_tile_size)))+ tOcO_i = cute.logical_divide(tOcO, cute.make_layout((self.m_block_size, corr_tile_size)))+ tOsO_i = cute.logical_divide(tOsO, cute.make_layout((self.m_block_size, corr_tile_size)))++ epi_subtile = (self.epi_tile[0], corr_tile_size)+ tmem_copy_atom = sm100_utils_basic.get_tmem_load_op(+ self.mma_tiler_pv,+ self.o_layout,+ self.o_dtype,+ self.pv_acc_dtype,+ epi_subtile,+ use_2cta_instrs=False,+ )+ tiled_tmem_load = tcgen05.make_tmem_copy(tmem_copy_atom, tOtO_i[(None, None), 0]).get_slice(+ tidx+ )+ thr_tmem_load = tiled_tmem_load.get_slice(tidx)+ smem_copy_atom = sm100_utils_basic.get_smem_store_op(+ self.o_layout, self.o_dtype, self.pv_acc_dtype, tiled_tmem_load+ )+ tiled_smem_store = cute.make_tiled_copy_D(smem_copy_atom, tiled_tmem_load)+ tOtO_t2r = thr_tmem_load.partition_S(tOtO_i[(None, None), None])+ tOsO_s2r = thr_tmem_load.partition_D(tOsO_i[(None, None), None])+ tOcO_t2r = thr_tmem_load.partition_D(tOcO_i[(None, None), None])+ for i in cutlass.range_constexpr(self.head_dim_v_padded // corr_tile_size):+ tOtO_t2r_i = tOtO_t2r[None, 0, 0, i]+ tOsO_r2s_i = tOsO_s2r[None, 0, 0, i]+ tOrO_frg = cute.make_rmem_tensor(tOcO_t2r[None, 0, 0, i].shape, self.pv_acc_dtype)+ cute.copy(tiled_tmem_load, tOtO_t2r_i, tOrO_frg)+ for j in cutlass.range_constexpr(0, cute.size(tOrO_frg), 2):+ tOrO_frg[j], tOrO_frg[j + 1] = utils.mul_packed_f32x2(+ (tOrO_frg[j], tOrO_frg[j + 1]),+ (scale, scale),+ )+ tOrO_frg_cvt = cute.make_rmem_tensor(tOrO_frg.shape, self.o_dtype)+ tOrO_frg_cvt.store(tOrO_frg.load().to(self.o_dtype))+ cute.copy(tiled_smem_store, tOrO_frg_cvt, tOsO_r2s_i)+ # fence view async shared+ cute.arch.fence_proxy("async.shared", space="cta")+ if const_expr(self.use_correction_warps_for_epi):+ assert(not self.use_tma_O)+ assert(gmem_tiled_copy_O is not None)+ cute.arch.barrier(barrier_id=int(NamedBarrierFwd.Epilogue),+ number_of_threads=len(self.epilogue_warp_ids) * cute.arch.WARP_SIZE)+ gmem_thr_copy_O = gmem_tiled_copy_O.get_slice(tidx)+ tOsO = gmem_thr_copy_O.partition_S(sO)+ cO = cute.make_identity_tensor((self.m_block_size, self.head_dim_v_padded))+ tOgO = gmem_thr_copy_O.partition_D(gO)+ tOcO = gmem_thr_copy_O.partition_S(cO)+ t0OcO = gmem_tiled_copy_O.get_slice(0).partition_S(cO)+ tOpO = utils.predicate_k(tOcO, limit=mO_cur.shape[1])+ # This output path does not support packed GQA.+ assert not self.pack_gqa+ pack_gqa = PackGQA(+ self.m_block_size,+ self.head_dim_v_padded,+ self.check_hdim_v_oob,+ self.qhead_per_kvhead,+ )++ # load acc O from smem to rmem for wider vectorization+ tOrO = cute.make_fragment_like(tOsO, self.o_dtype)+ cute.autovec_copy(tOsO, tOrO)+ effective_seqlen_q_corr = (+ Int32(1)+ if const_expr(self.seqlen_q_static_one)+ else seqlen_q+ )+ if const_expr(not self.pack_gqa):+ for rest_m in cutlass.range_constexpr(cute.size(tOrO.shape[1])):+ if (+ t0OcO[0, rest_m, 0][0]+ < effective_seqlen_q_corr+ - (self.q_stage * m_block + stage) * self.m_block_size+ - tOcO[0][0]+ ):+ cute.copy(+ gmem_tiled_copy_O,+ tOrO[None, rest_m, None],+ tOgO[None, rest_m, None, self.q_stage * m_block + stage],+ pred=tOpO[None, rest_m, None]+ if const_expr(self.check_hdim_v_oob)+ else None,+ )+ else:+ pack_gqa.store_O(+ mO_cur,+ tOrO,+ gmem_tiled_copy_O,+ tidx,+ self.q_stage * m_block + stage,+ effective_seqlen_q_corr,+ )++ @cute.jit+ def epilogue_s2g(+ self,+ mO: cute.Tensor,+ sO: cute.Tensor,+ gmem_tiled_copy_O: cute.TiledCopy,+ tma_atom_O: Optional[cute.CopyAtom],+ mbar_ptr: cute.Pointer,+ block_info: BlockInfo,+ num_splits: int,+ SeqlenInfoCls: Callable,+ TileSchedulerCls: Callable,+ mOutIndices: Optional[cute.Tensor] = None,+ ):+ epi_consumer_phase = Int32(0)+ tile_scheduler = TileSchedulerCls()+ work_tile = tile_scheduler.initial_work_tile_info()+ while work_tile.is_valid_tile:+ m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx+ seqlen = SeqlenInfoCls(batch_idx)+ n_block_min, n_block_max = self.split_block_range(+ block_info, seqlen, m_block, split_idx, num_splits+ )++ if self.tile_has_work(n_block_min, n_block_max, split_idx, num_splits):+ if const_expr(self.use_out_indices):+ mO_cur = seqlen.offset_batch_O_via_indices(+ mO, batch_idx, mOutIndices, dim=3,+ )[None, None, head_idx]+ elif const_expr(self.is_split_kv):+ mO_cur = seqlen.offset_batch_Q(mO, batch_idx, dim=3)[None, None, head_idx, split_idx]+ else:+ mO_cur = seqlen.offset_batch_Q(mO, batch_idx, dim=3)[None, None, head_idx]+ gO = cute.local_tile(mO_cur, (self.m_block_size, self.head_dim_v_padded), (None, 0))+ if const_expr(self.use_tma_O):+ store_O, _, _ = copy_utils.tma_get_copy_fn(+ tma_atom_O, 0, cute.make_layout(1), sO, gO+ )+ for stage in cutlass.range_constexpr(self.q_stage):+ # wait from corr, issue tma store on smem+ # 1. wait for O0 / O1 final+ iket.range_push("epi_wait_corr")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_corr_epi_full_offset + stage, epi_consumer_phase+ )+ iket.range_pop()+ # 2. copy O0 / O1 to gmem+ store_O(src_idx=stage, dst_idx=self.q_stage * m_block + stage)+ cute.arch.cp_async_bulk_commit_group()+ for stage in cutlass.range_constexpr(self.q_stage):+ # Ensure O0 / O1 buffer is ready to be released+ cute.arch.cp_async_bulk_wait_group(1 - stage, read=True)+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_corr_epi_empty_offset + stage)+ else:+ tidx = cute.arch.thread_idx()[0] % (+ cute.arch.WARP_SIZE * len(self.epilogue_warp_ids)+ )+ gmem_thr_copy_O = gmem_tiled_copy_O.get_slice(tidx)+ tOsO = gmem_thr_copy_O.partition_S(sO)+ cO = cute.make_identity_tensor((self.m_block_size, self.head_dim_v_padded))+ tOgO = gmem_thr_copy_O.partition_D(gO)+ tOcO = gmem_thr_copy_O.partition_S(cO)+ t0OcO = gmem_tiled_copy_O.get_slice(0).partition_S(cO)+ tOpO = utils.predicate_k(tOcO, limit=mO.shape[1])+ if const_expr(not self.use_out_indices and not self.seqlen_q_static_one):+ assert not self.pack_gqa+ pack_gqa = PackGQA(+ self.m_block_size,+ self.head_dim_v_padded,+ self.check_hdim_v_oob,+ self.qhead_per_kvhead,+ )+ # With seqlen_q statically one, PackGQA folds exactly+ # qhead_per_kvhead rows, and since that is at most one+ # m_block_size there is a single m_block and every reachable+ # row lives in Q stage 0. Both the row bound and the dead+ # stage are therefore compile-time constants.+ reachable_rows = None+ if const_expr(self.pack_gqa and self.seqlen_q_static_one):+ assert self.qhead_per_kvhead <= self.m_block_size+ # Replication spreads the same heads over the whole tile,+ # one every q_replicate rows, so the last reachable row+ # moves to the end of the tile.+ reachable_rows = self.qhead_per_kvhead * self.q_replicate+ for stage in cutlass.range_constexpr(self.q_stage):+ # wait from corr, issue tma store on smem+ # 1. wait for O0 / O1 final+ iket.range_push("epi_wait_corr")+ cute.arch.mbarrier_wait(+ mbar_ptr + self.mbar_corr_epi_full_offset + stage, epi_consumer_phase+ )+ iket.range_pop()+ # 2. copy O0 / O1 to gmem+ stage_is_live = (+ reachable_rows is None+ or stage * self.m_block_size < reachable_rows+ )+ if const_expr(stage_is_live):+ if const_expr(not self.pack_gqa):+ # load acc O from smem to rmem for wider vectorization+ tOrO = cute.make_fragment_like(+ tOsO[None, None, None, 0], self.o_dtype+ )+ cute.autovec_copy(tOsO[None, None, None, stage], tOrO)+ effective_seqlen_q_nonpack = (+ Int32(1)+ if const_expr(self.seqlen_q_static_one)+ else seqlen.seqlen_q+ )+ for rest_m in cutlass.range_constexpr(cute.size(tOrO.shape[1])):+ if (+ t0OcO[0, rest_m, 0][0]+ < effective_seqlen_q_nonpack+ - (self.q_stage * m_block + stage) * self.m_block_size+ - tOcO[0][0]+ ):+ cute.copy(+ gmem_tiled_copy_O,+ tOrO[None, rest_m, None],+ tOgO[+ None,+ rest_m,+ None,+ self.q_stage * m_block + stage,+ ],+ pred=tOpO[None, rest_m, None]+ if const_expr(self.check_hdim_v_oob)+ else None,+ )+ else:+ effective_seqlen_q = (+ Int32(1)+ if const_expr(self.use_out_indices or self.seqlen_q_static_one)+ else seqlen.seqlen_q+ )+ pack_gqa.store_O(+ mO_cur,+ tOsO[None, None, None, stage],+ gmem_tiled_copy_O,+ tidx,+ self.q_stage * m_block + stage,+ effective_seqlen_q,+ max_rows=reachable_rows,+ stage_from_smem=True,+ row_stride=self.q_replicate,+ )+ cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_corr_epi_empty_offset + stage)++ epi_consumer_phase ^= 1++ # Advance to next tile+ tile_scheduler.advance_to_next_work()+ work_tile = tile_scheduler.get_current_work()++ def load_Q(+ self,+ load_Q_fn: Callable,+ mbar_full_ptr: cute.Pointer,+ mbar_empty_ptr: cute.Pointer,+ block: Int32,+ stage: int,+ phase: Int32,+ load_SFQ_fn: Optional[Callable] = None,+ load_Q_second_fn: Optional[Callable] = None,+ load_SFQ_second_fn: Optional[Callable] = None,+ ):+ iket.range_push("load_wait_q")+ cute.arch.mbarrier_wait(mbar_empty_ptr + stage, phase)+ iket.range_pop()+ with cute.arch.elect_one():+ cute.arch.mbarrier_arrive_and_expect_tx(mbar_full_ptr + stage, self.tma_copy_bytes["Q"])+ load_Q_fn(+ src_idx=block,+ dst_idx=stage,+ tma_bar_ptr=mbar_full_ptr + stage,+ )+ if const_expr(load_SFQ_fn is not None):+ load_SFQ_fn(+ src_idx=block,+ dst_idx=stage,+ tma_bar_ptr=mbar_full_ptr + stage,+ )+ if const_expr(self.query_terms == 2):+ load_Q_second_fn(+ src_idx=block,+ dst_idx=stage,+ tma_bar_ptr=mbar_full_ptr + stage,+ )+ if const_expr(load_SFQ_second_fn is not None):+ load_SFQ_second_fn(+ src_idx=block,+ dst_idx=stage,+ tma_bar_ptr=mbar_full_ptr + stage,+ )++ @cute.jit+ def stack_Q_fp4_terms(+ self,+ sQ_uint8: cute.Tensor,+ sQ_second: cute.Tensor,+ sSFQ: cute.Tensor,+ sSFQ_second: cute.Tensor,+ mbar_full_ptr: cute.Pointer,+ mbar_ready_ptr: cute.Pointer,+ stage: int,+ phase: Int32,+ tidx: Int32,+ ):+ """Put the second term after the first term's live rows in SMEM.++ Both full query tiles and both scale tiles have already arrived by+ the time the load barrier completes. Their sizes and TMA transaction+ counts stay unchanged. Only padding in the first tile is replaced;+ the caller-owned scratch and its zero padding are never modified.+ One load warp copies complete bytes and scales, fences its writes for+ the Tensor Core/S2T consumers, then publishes one ready arrival.+ """+ assert self.q_stage == 1+ assert self.head_dim_padded == 128+ cute.arch.mbarrier_wait(mbar_full_ptr + stage, phase)+ rows = const_expr(self.transposed_query_rows)+ # Pointer swizzles act on byte addresses, so the word view retains+ # the byte view's SW(2,4,3); only its layout strides change units.+ # Each aligned word preserves eight packed nibbles. Two rows fill a+ # warp, with sixteen lanes moving the sixteen words of each row.+ q_word_layout = cute.make_layout(+ (self.m_block_size, 16), stride=(16, 1)+ )+ q_word_swizzle = cute.make_swizzle(2, 4, 3)+ src_q = cute.make_tensor(+ cute.recast_ptr(sQ_second.iterator, q_word_swizzle, cutlass.Uint32),+ q_word_layout,+ )+ dst_q = cute.make_tensor(+ cute.recast_ptr(sQ_uint8.iterator, q_word_swizzle, cutlass.Uint32),+ q_word_layout,+ )+ for row_pair in cutlass.range_constexpr(rows // 2):+ row = 2 * row_pair + tidx // 16+ word = tidx % 16+ dst_q[rows + row, word] = src_q[row, word]+ src_sf = cute.recast_tensor(+ sSFQ_second[None, None, None, stage], cutlass.Uint8+ )+ dst_sf = cute.recast_tensor(+ sSFQ[None, None, None, stage], cutlass.Uint8+ )+ # The scale layout is unchanged; only lane ownership changes, from+ # one row's lanes to a warp covering several rows at once. A row+ # holds head_dim / sf_vec_size scales, sf_groups_per_mma_k per MMA K+ # step: eight and four for NVFP4, four and two for MXFP4.+ scale_count = const_expr(+ rows * self.head_dim_padded // self.sf_vec_size+ )+ for scale_group in cutlass.range_constexpr(+ -(-scale_count // cute.arch.WARP_SIZE)+ ):+ scale_index = scale_group * cute.arch.WARP_SIZE + tidx+ if const_expr(scale_count % cute.arch.WARP_SIZE):+ # Fewer scales than lanes (MXFP4 at four rows): a spare lane+ # repeats another lane's copy, the same byte to the same+ # address.+ scale_index = scale_index % scale_count+ row = scale_index % rows+ scale = scale_index // rows+ mma_k = scale // self.sf_groups_per_mma_k+ k_inst = scale % self.sf_groups_per_mma_k+ dst = rows + row+ dst_sf[(((dst % 32, dst // 32), 0), (0, k_inst)), 0, mma_k] = (+ src_sf[(((row % 32, row // 32), 0), (0, k_inst)), 0, mma_k]+ )+ cute.arch.fence_view_async_shared()+ cute.arch.sync_warp()+ with cute.arch.elect_one():+ cute.arch.mbarrier_arrive(mbar_ready_ptr + stage)++ @cute.jit+ def replicate_Q_fp4_rows(+ self,+ sQ_uint8: cute.Tensor,+ sSFQ: cute.Tensor,+ mbar_full_ptr: cute.Pointer, # mbar_load_q_full (FP4 TMA arrival)+ mbar_ready_ptr: cute.Pointer, # mbar_q_fp4_ready+ stage: int,+ phase: Int32,+ tidx: Int32,+ ):+ """Repeat each query row across its block of the Q tile, in place.++ The TMA lands ``qhead_per_kvhead`` real rows and zero-fills the rest.+ This rewrites row ``m`` from row ``m // q_replicate`` so every row of the+ tile carries a query, which is what lets a softmax thread own a short+ column range instead of a whole row.++ Source rows are also destinations — row 1 feeds the second block and is+ itself overwritten from row 0 — so the order matters. Each thread owns a+ fixed set of byte columns for the whole tile, which removes any+ cross-thread hazard, and writes block 0 last, after every other block+ has consumed the source rows that live inside it.+ """+ cute.arch.mbarrier_wait(mbar_full_ptr + stage, phase)++ replicate = const_expr(self.q_replicate)+ real_rows = const_expr(self.m_block_size // replicate)+ bytes_per_row = const_expr(self.head_dim_padded // 2) # two FP4 per byte+ sf_groups_per_row = const_expr(self.head_dim_padded // self.sf_vec_size)+ num_threads = const_expr(len(self.load_warp_ids) * cute.arch.WARP_SIZE)+ assert bytes_per_row % num_threads == 0++ sQ_u8_stg = sQ_uint8[None, None, None, stage]+ sSFQ_u8_stg = cute.recast_tensor(sSFQ[None, None, None, stage], cutlass.Uint8)+ cols_per_thread = const_expr(bytes_per_row // num_threads)++ # Descending so that block 0, the one holding every source row, is+ # written after all of them have been read.+ for r_rev in cutlass.range_constexpr(real_rows):+ r = const_expr(real_rows - 1 - r_rev)+ for c in cutlass.range_constexpr(cols_per_thread):+ k_byte = tidx * cols_per_thread + c+ atom_k = k_byte % 32+ kH = k_byte // 32+ val = sQ_u8_stg[(r, atom_k), 0, kH]+ for i in cutlass.range_constexpr(replicate):+ sQ_u8_stg[(r * replicate + i, atom_k), 0, kH] = val+ if tidx < sf_groups_per_row:+ mma_k = tidx // 4+ k_inst = tidx % 4+ sf_val = sSFQ_u8_stg[(((r % 32, r // 32), 0), (0, k_inst)), 0, mma_k]+ for i in cutlass.range_constexpr(replicate):+ m = r * replicate + i+ sSFQ_u8_stg[(((m % 32, m // 32), 0), (0, k_inst)), 0, mma_k] = sf_val++ cute.arch.fence_view_async_shared()+ cute.arch.sync_warp()+ with cute.arch.elect_one():+ cute.arch.mbarrier_arrive(mbar_ready_ptr + stage)++ @cute.jit+ def load_KV(+ self,+ tma_atom: Optional[cute.CopyAtom],+ tXgX: Optional[cute.Tensor],+ tXsX: Optional[cute.Tensor],+ paged_kv_manager: Optional[PagedKVManager],+ sX: cute.Tensor,+ mbar_full_ptr: cute.Pointer,+ mbar_empty_ptr: cute.Pointer,+ block: Int32,+ producer_state: cutlass.pipeline.PipelineState,+ K_or_V: Literal["K", "V"],+ page_idx: Optional[Int32] = None,+ tma_atom_sf: Optional[cute.CopyAtom] = None,+ tXgSF: Optional[cute.Tensor] = None,+ tXsSF: Optional[cute.Tensor] = None,+ ):+ assert K_or_V in ("K", "V")+ stage, phase = producer_state.index, producer_state.phase+ iket.range_push("load_wait_kv")+ cute.arch.mbarrier_wait(mbar_empty_ptr + stage, phase)+ if const_expr(K_or_V == "K" and self.uneven_kv_smem):+ # Before this round, the smem location was occupied by V, which is smaller than+ # K. So we need to wait for the stage after that (stage 1) to be empty as well.+ if stage == 0:+ cute.arch.mbarrier_wait(mbar_empty_ptr + 1, phase)+ iket.range_pop()++ if const_expr(self.use_tma_KV):+ assert (+ tXgX is not None and+ tXsX is not None and+ tma_atom is not None+ )+ with cute.arch.elect_one():+ cute.arch.mbarrier_arrive_and_expect_tx(+ mbar_full_ptr + stage, self.tma_copy_bytes[K_or_V],+ )+ tXsX_cur = tXsX[None, stage]+ if const_expr(self.uneven_kv_smem):+ # Since this is the producer_state, the phase starts at 1, so we have to invert it+ tXsX_cur = self.offset_kv_smem(tXsX_cur, stage, phase ^ 1)+ # Currently we assume that page_size == n_block_size so we index into tXgX with block = 0+ tXgX_cur = tXgX[None, block] if const_expr(page_idx is None) else tXgX[None, 0, page_idx]+ cute.copy(tma_atom, tXgX_cur, tXsX_cur, tma_bar_ptr=mbar_full_ptr + stage)+ else:+ assert paged_kv_manager is not None+ paged_kv_manager.load_KV(block, sX[None, None, None, stage], K_or_V)+ cute.arch.cp_async_commit_group()+ cute.arch.cp_async_mbarrier_arrive_noinc(mbar_full_ptr + stage)++ # Load scale factor for K or V if provided (uses same barrier as K/V)+ if const_expr(tma_atom_sf is not None and tXgSF is not None and tXsSF is not None):+ tXsSF_cur = tXsSF[None, stage]+ # After tma_partition with rank-1 grouping, tXgSF has structure: ((atom_v, rest_v), RestL)+ if const_expr(page_idx is None):+ tXgSF_cur = tXgSF[None, block]+ else:+ tXgSF_cur = tXgSF[None, 0, page_idx]+ cute.copy(tma_atom_sf, tXgSF_cur, tXsSF_cur, tma_bar_ptr=mbar_full_ptr + stage)++ def mainloop_s2t_copy_and_partition(+ self,+ sSF: cute.Tensor,+ tSF: cute.Tensor,+ ) -> Tuple[cute.TiledCopy, cute.Tensor, cute.Tensor]:+ """+ Make tiledCopy for smem to tmem load for scale factor tensor, then use it to partition smem memory (source) and tensor memory (destination).++ :param sSF: The scale factor tensor in smem+ :type sSF: cute.Tensor+ :param tSF: The scale factor tensor in tmem+ :type tSF: cute.Tensor++ :return: A tuple containing (tiled_copy_s2t, tCsSF_compact_s2t, tCtSF_compact_s2t) where:+ - tiled_copy_s2t: The tiled copy operation for smem to tmem load for scale factor tensor(s2t)+ - tCsSF_compact_s2t: The partitioned scale factor tensor in smem+ - tCtSF_compact_s2t: The partitioned scale factor tensor in tmem+ :rtype: Tuple[cute.TiledCopy, cute.Tensor, cute.Tensor]+ """+ # (MMA, MMA_MN, MMA_K, STAGE)+ tCsSF_compact = cute.filter_zeros(sSF)+ # (MMA, MMA_MN, MMA_K)+ tCtSF_compact = cute.filter_zeros(tSF)++ # Make S2T CopyAtom and tiledCopy+ copy_atom_s2t = cute.make_copy_atom(+ tcgen05.Cp4x32x128bOp(self.cta_group),+ self.sf_dtype,+ )+ tiled_copy_s2t = tcgen05.make_s2t_copy(copy_atom_s2t, tCtSF_compact)+ thr_copy_s2t = tiled_copy_s2t.get_slice(0)++ # ((ATOM_V, REST_V), Rest_Tiler, MMA_MN, MMA_K, STAGE)+ tCsSF_compact_s2t_ = thr_copy_s2t.partition_S(tCsSF_compact)+ # ((ATOM_V, REST_V), Rest_Tiler, MMA_MN, MMA_K, STAGE)+ tCsSF_compact_s2t = tcgen05.get_s2t_smem_desc_tensor(+ tiled_copy_s2t, tCsSF_compact_s2t_+ )+ # ((ATOM_V, REST_V), Rest_Tiler, MMA_MN, MMA_K)+ tCtSF_compact_s2t = thr_copy_s2t.partition_D(tCtSF_compact)+ return tiled_copy_s2t, tCsSF_compact_s2t, tCtSF_compact_s2t++ @cute.jit+ def offset_kv_smem(self, sX: cute.Tensor, stage: Int32, phase: Int32):+ if const_expr(self.uneven_kv_smem):+ # smem layout is [smem_large, smem_small, smem_large], and the current stride is+ # (smem_large + smem_small) // 2. So for stage == 1, move right by offset if+ # phase == 0, or left by offset if phase == 1.+ offset = 0 if stage != 1 else self.uneven_kv_smem_offset * (1 - 2 * phase)+ return cute.make_tensor(sX.iterator + offset, sX.layout)+ else:+ return sX++ def make_and_init_load_kv_pipeline(self, load_kv_mbar_ptr, use_k_bytes: bool = True):+ load_kv_consumer_group = cutlass.pipeline.CooperativeGroup(+ cutlass.pipeline.Agent.Thread, len([self.mma_warp_id])+ )+ if self.use_tma_KV:+ load_kv_producer_group = cutlass.pipeline.CooperativeGroup(+ cutlass.pipeline.Agent.Thread, len(self.load_warp_ids)+ )+ return cutlass.pipeline.PipelineTmaUmma.create(+ barrier_storage=load_kv_mbar_ptr,+ num_stages=self.kv_stage,+ producer_group=load_kv_producer_group,+ consumer_group=load_kv_consumer_group,+ tx_count=self.tma_copy_bytes["K"] if use_k_bytes else self.tma_copy_bytes["V"],+ )+ else:+ load_kv_producer_group = cutlass.pipeline.CooperativeGroup(+ cutlass.pipeline.Agent.Thread, len(self.load_warp_ids) * cute.arch.WARP_SIZE+ )+ return cutlass.pipeline.PipelineAsyncUmma.create(+ num_stages=self.kv_stage,+ producer_group=load_kv_producer_group,+ consumer_group=load_kv_consumer_group,+ barrier_storage=load_kv_mbar_ptr,+ )++ # @cute.jit+ # def warp_scheduler_barrier_init(self):+ # warp_group_idx = utils.canonical_warp_group_idx(sync=False)+ # if warp_group_idx == 0:+ # cute.arch.barrier_arrive(+ # barrier_id=int(NamedBarrierFwd.WarpSchedulerWG1), number_of_threads=2 * 128,+ # )++ # def warp_scheduler_barrier_sync(self):+ # cute.arch.barrier(+ # barrier_id=int(NamedBarrierFwd.WarpSchedulerWG1) + utils.canonical_warp_group_idx(sync=False),+ # number_of_threads=2 * 128+ # )++ # def warp_scheduler_barrier_arrive(self):+ # cur_wg = utils.canonical_warp_group_idx(sync=False)+ # next_wg = 1 - cur_wg+ # cute.arch.barrier_arrive(+ # barrier_id=int(NamedBarrierFwd.WarpSchedulerWG1) + next_wg, number_of_threads=2 * 128,+ # )++ @cute.jit+ def apply_score_mod(+ self,+ tSrS_t2r,+ thr_tmem_load,+ thr_mma_qk,+ batch_idx,+ head_idx,+ m_block,+ n_block,+ softmax,+ aux_tensors=None,+ fastdiv_mods=(None, None),+ ):+ """Apply score modification for SM100 (constant q_idx)."""+ # Prepare index tensor with extra partition+ cS = cute.make_identity_tensor((self.m_block_size, self.n_block_size))+ cS = cute.domain_offset((m_block * self.m_block_size, n_block * self.n_block_size), cS)+ tScS = thr_mma_qk.partition_C(cS)+ tScS_t2r = thr_tmem_load.partition_D(tScS)++ # Shared q_idx for all scores+ q_idx_logical = tScS_t2r[0][0]++ # For Pack-GQA, compute the logical head index for this tile+ if cutlass.const_expr(self.pack_gqa):+ # Building up the logical q_head idx: final_q_head = kv_head * qhead_per_kvhead + (q_physical % qhead_per_kvhead)+ q_physical = q_idx_logical+ q_idx_logical = q_physical // self.qhead_per_kvhead+ head_offset = q_physical - q_idx_logical * self.qhead_per_kvhead+ head_idx = head_idx * self.qhead_per_kvhead + head_offset++ if cutlass.const_expr(aux_tensors is not None):+ seqlen_q_divmod, _ = fastdiv_mods+ _, q_idx_logical = seqlen_q_divmod.divmod(q_idx_logical)++ apply_score_mod_inner(+ tSrS_t2r,+ tScS_t2r,+ self.score_mod,+ batch_idx,+ head_idx,+ softmax.softmax_scale,+ self.vec_size,+ self.qk_acc_dtype,+ aux_tensors,+ fastdiv_mods,+ constant_q_idx=q_idx_logical,+ qhead_per_kvhead=self.qhead_per_kvhead if cutlass.const_expr(self.pack_gqa) else 1,+ )diff --git a/src/nvfp4_decode_kernel/interface.py b/src/nvfp4_decode_kernel/interface.pynew file mode 100644index 0000000..974e2b3--- /dev/null+++ b/src/nvfp4_decode_kernel/interface.py@@ -0,0 +1,312 @@+"""Framework-independent interface for paged NVFP4 and MXFP4 decode."""++from __future__ import annotations++from typing import NamedTuple++import torch++from .scale_mode import DEFAULT_SCALE_MODE+++# Rows of the tile the residual MMA reads the BF16 query through. Only row 0 of+# each tile holds a real query, since decode has one token per row.+RESIDUAL_ROW_TILE = 128+# Leading tokens retained in the BF16 cache. FP4 keeps its original page origin+# and page zero's duplicate is masked during decode.+SINK_TOKENS = 16+++class KernelNotAvailableError(RuntimeError):+ """Raised when the kernel implementation has not been installed."""+++def split_partial_count(+ rows: int, heads_kv: int, num_splits: int, *, residual: bool, sms: int+) -> int:+ """How many partial outputs per row a decode at ``num_splits`` writes.++ It is ``num_splits``, one per partition of the FP4 pages, plus one when a+ residual is fused and the tail and the sink get a split of their own. They+ get one only while the launch still fits on the device's ``sms`` SMs in a+ single wave with it, ``rows * heads_kv * (num_splits + 1) <= sms``: the+ extra CTA is worth having because it runs beside the FP4 splits and takes+ the BF16 blocks off every one of them, not after them. Otherwise every+ split runs the BF16 blocks and only split 0 counts them.++ ``split_partial_scratch`` and ``split_lse_scratch`` are sized by this.+ """+ if num_splits <= 1:+ return num_splits+ own_split = residual and rows * heads_kv * (num_splits + 1) <= sms+ return num_splits + int(own_split)+++class DecodeTailWrite(NamedTuple):+ """This step's K/V token of each decode row, bound for the BF16 tail.++ ``key`` and ``value`` are ``[>= rows, heads_kv, head_dim]`` BF16 with row+ ``i`` the token of decode row ``i``, at position ``seq_lens[i] - 1``. It+ lands at ``position - seqused_fp4[i]`` of the rolling tail, and also at+ ``RESIDUAL_ROW_TILE + position`` while the position is below+ ``sink_tokens``. ``tail_key`` / ``tail_value`` are one layer's+ ``[slots, RESIDUAL_ROW_TILE + sink_tokens, heads_kv, head_dim]``, the+ slot is ``row_to_slot[i]``, and a negative slot writes nothing. K is+ stored in the cache's basis: Hadamard-rotated when ``rotate_key``, then+ ``key_shift`` (FP32 ``[heads_kv, head_dim]``, already in the rotated+ basis) subtracted, then rounded to BF16 once.+ """++ key: torch.Tensor+ value: torch.Tensor+ tail_key: torch.Tensor+ tail_value: torch.Tensor+ row_to_slot: torch.Tensor+ seq_lens: torch.Tensor+ seqused_fp4: torch.Tensor+ rotate_key: bool+ key_shift: torch.Tensor | None+ sink_tokens: int+++def fp4_decode(+ query: torch.Tensor | None = None,+ key_pages_fp4: torch.Tensor | None = None,+ key_scales: torch.Tensor | None = None,+ value_pages_fp4: torch.Tensor | None = None,+ value_scales: torch.Tensor | None = None,+ fp4_page_table: torch.Tensor | None = None,+ seqused_fp4: torch.Tensor | None = None,+ *,+ query_fp4: torch.Tensor | None = None,+ query_scales: torch.Tensor | None = None,+ query_fp4_second: torch.Tensor | None = None,+ query_scales_second: torch.Tensor | None = None,+ residual_key_pages_bf16: torch.Tensor | None = None,+ residual_value_pages_bf16: torch.Tensor | None = None,+ residual_page_ids: torch.Tensor | None = None,+ seqused_residual: torch.Tensor | None = None,+ seqused_sink: torch.Tensor | None = None,+ has_bf16: torch.Tensor | None = None,+ softmax_scale: float | None = None,+ query_row_indices: torch.Tensor | None = None,+ out: torch.Tensor | None = None,+ out_indices: torch.Tensor | None = None,+ trusted_metadata: bool = False,+ num_splits: int = 1,+ query_padded_scratch: torch.Tensor | None = None,+ query_fp4_scratch: torch.Tensor | None = None,+ query_scales_scratch: torch.Tensor | None = None,+ query_fp4_second_scratch: torch.Tensor | None = None,+ query_scales_second_scratch: torch.Tensor | None = None,+ split_partial_scratch: torch.Tensor | None = None,+ split_lse_scratch: torch.Tensor | None = None,+ query_terms: int = 1,+ scale_mode: str = DEFAULT_SCALE_MODE,+ query_rotation: str = "none",+ tail_write: DecodeTailWrite | None = None,+) -> torch.Tensor:+ """Run paged FP4 decode attention.++ Args:+ query: Optional BF16 query in ``[rows, heads_q, head_dim]`` layout. It may+ contain more rows than the current decode batch when+ ``query_row_indices`` is provided. Supply either this argument or+ both ``query_fp4`` and ``query_scales``.+ query_fp4: Optional pre-quantized packed E2M1 query from+ ``quantize_query``.+ query_scales: Optional E4M3 scale-factor bytes paired with+ ``query_fp4``.+ query_fp4_second: Optional residual-query term paired with+ ``query_scales_second``. Both are required when using the+ pre-quantized two-term path.+ key_pages_fp4: Packed E2M1 K pages.+ key_scales: E4M3 (NVFP4) or E8M0 (MXFP4) scale factors for ``key_pages_fp4``.+ value_pages_fp4: Packed E2M1 V pages.+ value_scales: E4M3 (NVFP4) or E8M0 (MXFP4) scale factors for ``value_pages_fp4``.+ fp4_page_table: INT32 logical-to-physical FP4 page mapping, shaped+ ``[rows, max_pages_per_row]``.+ seqused_fp4: INT32 FP4 token count for each row, shaped ``[rows]``.+ Every value must be a multiple of the page size. With an attention+ sink this remains the prefix on the original page grid; page zero+ may therefore contain a masked copy of the sink.+ residual_key_pages_bf16: Optional physical BF16 K-page cache.+ residual_value_pages_bf16: Optional physical BF16 V-page cache.+ residual_page_ids: Optional INT32 physical BF16 page ID per row.+ seqused_residual: Optional INT32 valid BF16 token count per row.+ seqused_sink: Optional INT32 valid BF16 attention-sink token count per+ row. When supplied, the sink occupies rows+ ``[RESIDUAL_ROW_TILE, RESIDUAL_ROW_TILE + SINK_TOKENS)`` of the+ same physical page selected by ``residual_page_ids``. The first+ ``seqused_sink`` positions of FP4 page zero are treated as+ duplicates and masked. Before page zero is promoted, the same+ prefix is masked in the rolling BF16 page instead.+ has_bf16: Optional BOOL indicator of whether each row owns an active+ BF16 residual page. It must agree with ``seqused_residual > 0``.+ softmax_scale: Score scale. The implementation uses+ ``head_dim**-0.5`` when omitted.+ query_row_indices: Optional INT32 mapping from compact decode rows to+ rows in a full 3-D query tensor.+ out: Optional BF16 output tensor in ``[rows, heads_q, head_dim]``+ layout, written in place instead of a fresh allocation. Row i of+ the batch lands in row i unless ``out_indices`` says otherwise.+ out_indices: Optional INT32 mapping from compact decode rows to rows+ in ``out``. Costs split-K, which cannot reorder rows on its way+ out, so pass it only when the rows really do have to move.+ num_splits: How many key partitions the decode is split across, as a+ positive integer. One CTA serves one row and KV head per+ split, so the count is what fills the device when the batch alone+ cannot. The right value depends on the batch and the SM count,+ neither of which this function knows, hence the default of one;+ ``nvfp4_vllm`` chooses it per step from the decode row count.+ Never derive it from the page table: under CUDA graph capture the+ table is frozen at the model's maximum length, and a count read+ from it would be baked into every later replay.++ The count partitions the FP4 pages. A fused residual's tail and+ sink are merged once, either by a split of their own launched+ beside them or by split 0; ``split_partial_count`` says which.++ A value above one is honoured only where the split path can serve+ it: no residual argument at all, or a complete residual on the+ BF16 query path, and never together with ``out_indices``, whose+ reordering the combine cannot perform.+ Asking for a split anywhere else raises, because a request quietly+ downgraded to one tile would look exactly like a split that ran.+ query_padded_scratch: Optional BF16 buffer shaped ``[at least rows,+ RESIDUAL_ROW_TILE, heads_q, head_dim]`` that holds the query padded+ to the residual MMA's row tile. Only the first row of each tile is+ written, so a caller-owned buffer has to be zeroed once and never+ again. Supplying one avoids reallocating and rezeroing it on every+ call, which matters when a residual is present on every layer of+ every step.+ query_fp4_scratch: Optional UINT8 buffer shaped ``[at least rows, 1,+ heads_q, head_dim // 2]`` that receives the packed E2M1 query. The+ quantizer rewrites every byte of it, so whatever it arrives+ holding is irrelevant.+ query_scales_scratch: Optional UINT8 buffer shaped ``[at least rows,+ 1, heads_q, head_dim // (4 * sf_vec_size), 32, 4, 4]`` that receives+ the query scale factors. ``sf_vec_size`` is 16 for NVFP4 and+ 32 for MXFP4. Unlike ``query_fp4_scratch`` this one carries+ the same obligation as ``query_padded_scratch``, and for a+ sharper reason: the layout reserves ``32 * 4`` scale slots per KV+ head and only the ``heads_q // heads_kv`` of them that carry a+ query head are ever written. Nothing clears the others, so a+ buffer zeroed once is byte-for-byte what an internal allocation+ would have produced, while one handed over uninitialized feeds the+ MMA whatever it found for as long as it is reused. No shape check+ can catch that.++ For the same reason a buffer belongs to one ``(heads_q, heads_kv,+ head_dim)`` triple for its whole life. Its shape says nothing+ about ``heads_kv``, so the shape check cannot notice the reuse,+ but ``heads_q // heads_kv`` is exactly what picks the slots that+ get written: at ``heads_kv == 1`` query head ``h`` lands in slot+ ``h`` of the single KV head's tile, and at ``heads_kv == 8`` in+ slot ``h % (heads_q // 8)`` of tile ``h // (heads_q // 8)``.+ Handing the first buffer to the second geometry leaves the slots+ only the first one wrote still holding its scales, where the+ second needs zeros.+ split_partial_scratch: Optional contiguous 1-D FP32 buffer, starting+ at storage offset zero, that holds the split path's partial+ outputs: at least ``split_partial_count(...) * rows * heads_q *+ head_dim`` elements, which with a residual can be one partial per+ row more than ``num_splits``. Unlike the query scratch it carries+ no obligation to be zeroed, then or ever: every split writes its+ LSE, an empty one -inf, and the combine adds a partial output only+ under a positive weight. Ignored at one split.+ split_lse_scratch: The same for the split path's log-sum-exps: at+ least ``split_partial_count(...) * rows * heads_q`` elements.+ Supplying both+ means a split decode allocates and fills nothing, which matters+ under CUDA graph capture, where both would otherwise be replayed+ on every layer of every step.+ scale_mode: Which block-scale candidates the query quantizer searches;+ see :mod:`.scale_mode`. Applies to the BF16 query path only — on+ the pre-quantized path the caller has already chosen, and naming a+ second mode here is refused rather than ignored. It does not+ describe the pages, which were written by whatever mode was in+ force then; nothing in the stored format records which.+ query_terms: One or two independently scaled NVFP4 query terms. Two+ quantizes the residual of the first term and accumulates both QK+ products before softmax.+ query_rotation: ``"hadamard"`` multiplies each BF16 query head by the+ normalized Sylvester Hadamard inside the quantizer, in FP32, and+ quantizes every term from those FP32 values; the BF16 query the+ residual MMA reads is the same rotation rounded once. The pages+ must then hold keys in the same basis. BF16 query path only, like+ ``scale_mode``.+ tail_write: Optional :class:`DecodeTailWrite`. The query quantizer+ then also copies each decode row's new K/V token into the BF16+ tail from extra CTAs of its own launch, so the decode launch that+ follows reads it with no separate write kernel ahead of them. It+ stores K as ``KVCacheWriter`` does, byte for byte, and is not a+ quantization: whole FP4 pages still come only from the serving+ code. BF16 query path only.++ Returns:+ A compact BF16 output for the selected query rows. It is a view of+ ``out`` when one was supplied rather than a fresh allocation: the+ whole of ``out`` when ``out_indices`` scattered the rows across it,+ and ``out[:rows]`` otherwise, because the rows past the batch belong+ to whoever else shares the buffer.+ """+ try:+ from ._kernel import fp4_decode_impl+ except ModuleNotFoundError as error:+ if error.name != f"{__package__}._kernel":+ raise+ raise KernelNotAvailableError(+ "NVFP4 kernel implementation is unavailable."+ ) from error++ required = {+ "key_pages_fp4": key_pages_fp4,+ "key_scales": key_scales,+ "value_pages_fp4": value_pages_fp4,+ "value_scales": value_scales,+ "fp4_page_table": fp4_page_table,+ "seqused_fp4": seqused_fp4,+ }+ missing = [name for name, value in required.items() if value is None]+ if missing:+ raise ValueError(+ f"missing required decode arguments: {', '.join(missing)}"+ )++ return fp4_decode_impl(+ query=query,+ query_fp4=query_fp4,+ query_scales=query_scales,+ query_fp4_second=query_fp4_second,+ query_scales_second=query_scales_second,+ key_pages_fp4=key_pages_fp4,+ key_scales=key_scales,+ value_pages_fp4=value_pages_fp4,+ value_scales=value_scales,+ fp4_page_table=fp4_page_table,+ seqused_fp4=seqused_fp4,+ residual_key_pages_bf16=residual_key_pages_bf16,+ residual_value_pages_bf16=residual_value_pages_bf16,+ residual_page_ids=residual_page_ids,+ seqused_residual=seqused_residual,+ seqused_sink=seqused_sink,+ has_bf16=has_bf16,+ softmax_scale=softmax_scale,+ query_row_indices=query_row_indices,+ out=out,+ out_indices=out_indices,+ trusted_metadata=trusted_metadata,+ num_splits=num_splits,+ query_padded_scratch=query_padded_scratch,+ query_fp4_scratch=query_fp4_scratch,+ query_scales_scratch=query_scales_scratch,+ query_fp4_second_scratch=query_fp4_second_scratch,+ query_scales_second_scratch=query_scales_second_scratch,+ split_partial_scratch=split_partial_scratch,+ split_lse_scratch=split_lse_scratch,+ query_terms=query_terms,+ scale_mode=scale_mode,+ query_rotation=query_rotation,+ tail_write=tail_write,+ )diff --git a/src/nvfp4_decode_kernel/p_scale.py b/src/nvfp4_decode_kernel/p_scale.pynew file mode 100644index 0000000..b9e93e3--- /dev/null+++ b/src/nvfp4_decode_kernel/p_scale.py@@ -0,0 +1,97 @@+"""Configuration for the ephemeral NVFP4 softmax-probability operand."""++from __future__ import annotations++import os+import math+from collections.abc import Mapping+++# Fallback threshold in scaled-probability units; calibration overrides it+# through NVFP4_P_SCALE_TAU. Using amax/2 for small groups keeps their E4M3+# scales farther from zero than amax/6. The default 21/1024 lies halfway+# between the adjacent E4M3 values 20/1024 and 22/1024.+DEFAULT_P_SCALE_TAU = 21.0 / 1024.0+++def resolve_p_scale_tau(environment: Mapping[str, str] | None = None) -> float:+ """Threshold in scaled-probability units: choose /2 when c * max(p) <= tau."""+ env = os.environ if environment is None else environment+ tau = float(env.get("NVFP4_P_SCALE_TAU", DEFAULT_P_SCALE_TAU))+ if not math.isfinite(tau) or not 0 <= tau <= 448:+ raise ValueError("NVFP4_P_SCALE_TAU must be finite and between 0 and 448")+ return tau+++# Read before kernel import: the comparison is embedded in its generated PTX.+P_SMALL_GROUP_AMAX = resolve_p_scale_tau()++P_SCALE_MODES: dict[str, tuple[float, ...]] = {+ "amax6": (6.0,),+ "two_level": (6.0,),+ "amax4": (4.0,),+ "amax3": (3.0,),+ "amax2": (2.0,),+ "e2m1_max_grid": (6.0, 4.0, 3.0, 2.0),+ "amax2_or_amax6": (2.0, 6.0),+ "amax2_or_amax6_c8": (2.0, 6.0),+}++DEFAULT_P_SCALE_MODE = "amax2_or_amax6_c8"++# Accept the older configuration name for ``amax2_or_amax6``.+_LEGACY_P_SCALE_MODES = {"e4m3_amax_code10_threshold": "amax2_or_amax6"}+++def p_scale_is_amax2_or_amax6(mode: str) -> bool:+ """Whether ``mode`` selects amax/2 for small groups and amax/6 otherwise."""+ return _LEGACY_P_SCALE_MODES.get(mode, mode) in {+ "amax2_or_amax6", "amax2_or_amax6_c8",+ }+++def resolve_p_scale_environment(+ environment: Mapping[str, str] | None = None,+) -> tuple[str, float, bool]:+ """Return ``(mode, fixed_divisor, searches_grid)`` with contradiction checks.++ With neither variable set the mode is ``DEFAULT_P_SCALE_MODE``.+ ``NVFP4_P_SCALE_DIVISOR`` selects the divisor for fixed modes. Adaptive+ modes return six as a constructor placeholder; a separate compile-time+ flag selects the actual divisor.+ """+ env = os.environ if environment is None else environment+ mode = env.get("NVFP4_P_SCALE_MODE")+ divisor_text = env.get("NVFP4_P_SCALE_DIVISOR")+ if mode is None and divisor_text is None:+ mode = DEFAULT_P_SCALE_MODE+ if mode is None:+ try:+ divisor = float(divisor_text)+ except ValueError as error:+ raise ValueError(+ "NVFP4_P_SCALE_DIVISOR must be 2, 3, 4, or 6, got "+ f"{divisor_text!r}"+ ) from error+ if divisor not in (2.0, 3.0, 4.0, 6.0):+ raise ValueError(+ "NVFP4_P_SCALE_DIVISOR must be 2, 3, 4, or 6, got "+ f"{divisor_text!r}"+ )+ return f"amax{int(divisor)}", divisor, False++ mode = _LEGACY_P_SCALE_MODES.get(mode, mode)+ if mode not in P_SCALE_MODES:+ raise ValueError(+ f"NVFP4_P_SCALE_MODE must be one of {sorted(P_SCALE_MODES)}, "+ f"got {mode!r}"+ )+ divisors = P_SCALE_MODES[mode]+ grid = mode == "e2m1_max_grid"+ divisor = divisors[0] if len(divisors) == 1 else 6.0+ if divisor_text is not None and float(divisor_text) != divisor:+ raise ValueError(+ f"NVFP4_P_SCALE_MODE={mode!r} contradicts "+ f"NVFP4_P_SCALE_DIVISOR={divisor_text!r}"+ )+ return mode, divisor, griddiff --git a/src/nvfp4_decode_kernel/quantize_kv_kernel.py b/src/nvfp4_decode_kernel/quantize_kv_kernel.pynew file mode 100644index 0000000..78b4647--- /dev/null+++ b/src/nvfp4_decode_kernel/quantize_kv_kernel.py@@ -0,0 +1,1184 @@+"""CuTeDSL K/V page quantization."""++import math+from typing import Tuple++import torch+import tvm_ffi++import cutlass+import cutlass.cute as cute+from cutlass import Int32, const_expr+from cutlass.cute.runtime import make_fake_stream, make_fake_tensor++from ._block_scale import (+ GROUPS_PER_ATOM,+ _load_vector,+ gaussian_global_scale,+ quantize_atom,+ quantize_atom_with_scale,+)+from .quantize_q_kernel import _mul_rn, _sub_rn, _write_tail_token+from .scale_mode import DEFAULT_SCALE_MODE, scale_format, scale_rule, sf_vec_size+++PAGE_SIZE = 128+HEAD_DIM = 128+SF_VEC_SIZE = 16+KEY_SCALE_GEOMETRIES = ("local", "gaussian_global")+++def _resolve_work(+ source_tokens: cute.Tensor,+ destination_pages: cute.Tensor,+ work: Int32,+) -> Tuple[Int32, Int32]:+ """Map a grid slot to the token it reads and the page it writes.++ The two ends are independent, which is what a paged cache needs — the+ tokens of one page are somewhere in a flat activation buffer and the page+ they belong to is wherever the block table says. A negative destination+ marks a grid slot with no work, so the launch shape can be fixed while the+ batch varies.+ """+ return source_tokens[work], destination_pages[work]+++def _rebase(+ tensor: cute.Tensor,+ bases,+ region: cutlass.Constexpr[int],+ layer: Int32,+) -> cute.Tensor:+ """The same page layout, based wherever this layer's copy happens to be.++ A caller with one destination passes no table and gets the tensor back+ untouched. A caller with one destination per layer passes their base+ addresses, because independently allocated destinations share only their+ interior layout — their addresses are not a stride apart, so no single+ tensor can reach all of them. Row ``region`` of the table is this tensor's.+ """+ if const_expr(bases is None):+ return tensor+ return cute.make_tensor(+ cute.make_ptr(+ cutlass.Uint8,+ bases[region, layer],+ cute.AddressSpace.gmem,+ assumed_align=16,+ ),+ tensor.layout,+ )+++def _quantize_vector(+ values: cute.Tensor,+ destination: cute.Tensor,+ scales: cute.Tensor,+ page: Int32,+ lane: Int32,+ warp: Int32,+ head: Int32,+ sf_vec_size: cutlass.Constexpr[int],+ rest_k: cutlass.Constexpr[int],+ rule: cutlass.Constexpr,+ use_gaussian_global_scale: cutlass.Constexpr[bool],+):+ """Quantize one thread's vector to E2M1 with one scale per group.++ K and V reduce to this once their page slicing is done: K walks the head+ dimension of one token, V walks one head dimension across the page. The+ scale tile is addressed identically in both, which is what keeps the two+ directions numerically the same.++ ``values`` holds the vector in registers.+ """+ vector_scale = None+ if const_expr(use_gaussian_global_scale):+ vector_scale = gaussian_global_scale(values)+ for k_atom in cutlass.range_constexpr(rest_k):+ if const_expr(use_gaussian_global_scale):+ packed_scale = quantize_atom_with_scale(+ values,+ destination,+ k_atom,+ sf_vec_size,+ vector_scale,+ )+ else:+ packed_scale = quantize_atom(+ values, destination, k_atom, sf_vec_size, rule+ )+ scale_tile = scales[page, lane, warp, 0, None, k_atom, head]+ scale_word = cute.make_tensor(+ cute.recast_ptr(scale_tile.iterator, dtype=cutlass.Int32),+ cute.make_layout(1),+ )+ scale_word[0] = packed_scale+++def _cache_basis(+ values: cute.Tensor,+ shift_vector: cute.Tensor,+ rotate: cutlass.Constexpr[bool],+) -> cute.Tensor:+ """One token's K head as the cache stores it: rotated, then centered.++ The exact BF16 values are widened to FP32 and butterflied ``(a, c) ->+ (a + c, a - c)`` over the index bits, lowest first; one separately rounded+ multiply by the FP32 ``1 / sqrt(128)`` and one separately rounded subtract+ of the shift (already in the rotated basis, BF16 widened exactly) follow,+ and the result is rounded to BF16 once. A butterfly's result depends only+ on its operands' order, not on which thread holds them, so this is the+ decode quantizer's ``_write_tail_token`` arithmetic element for element.+ A contracted FMA in place of either rounded step moves 7 in 307,200 values+ by one BF16 step.+ """+ work = cute.make_rmem_tensor(cute.make_layout(HEAD_DIM), cutlass.Float32)+ work.store(values.load().to(cutlass.Float32))+ if const_expr(rotate):+ for bit in cutlass.range_constexpr(int(math.log2(HEAD_DIM))):+ stride = 1 << bit+ for j in cutlass.range_constexpr(HEAD_DIM):+ if const_expr((j & stride) == 0):+ low = work[j]+ high = work[j + stride]+ work[j] = low + high+ work[j + stride] = low - high+ inverse = cutlass.Float32(1.0 / math.sqrt(HEAD_DIM))+ for j in cutlass.range_constexpr(HEAD_DIM):+ work[j] = _mul_rn(work[j], inverse)+ if const_expr(shift_vector is not None):+ centre = cute.make_fragment_like(shift_vector)+ cute.autovec_copy(shift_vector, centre)+ for j in cutlass.range_constexpr(HEAD_DIM):+ work[j] = _sub_rn(work[j], cutlass.Float32(centre[j]))+ rounded = cute.make_rmem_tensor(cute.make_layout(HEAD_DIM), cutlass.BFloat16)+ rounded.store(work.load().to(cutlass.BFloat16))+ return rounded+++def _key_page(+ key: cute.Tensor,+ key_shift: cute.Tensor,+ key_pages_fp4: cute.Tensor,+ key_scales: cute.Tensor,+ source_tokens: cute.Tensor,+ destination_pages: cute.Tensor,+ destination_bases: cute.Tensor,+ item: Int32,+ layer: Int32,+ source_offset: Int32,+ heads: cutlass.Constexpr[int],+ sf_vec_size: cutlass.Constexpr[int],+ rest_k: cutlass.Constexpr[int],+ rule: cutlass.Constexpr,+ use_gaussian_global_scale: cutlass.Constexpr[bool],+ rotate: cutlass.Constexpr[bool],+):+ """One head of one K page, one token's head vector per thread."""+ thread, _, _ = cute.arch.thread_idx()+ lane = thread % 32+ warp = thread // 32+ head = item % heads+ source_token, page = _resolve_work(+ source_tokens, destination_pages, item // heads+ )+ if page >= 0:+ source = key[source_token + source_offset + thread, head, None]+ values = _load_vector(source)+ if const_expr(rotate or key_shift is not None):+ shift_vector = None+ if const_expr(key_shift is not None):+ shift_vector = key_shift[layer, head, None]+ values = _cache_basis(values, shift_vector, rotate)+ _quantize_vector(+ values,+ _rebase(key_pages_fp4, destination_bases, 0, layer)[+ page, thread, head, None+ ],+ _rebase(key_scales, destination_bases, 1, layer),+ page,+ lane,+ warp,+ head,+ sf_vec_size,+ rest_k,+ rule,+ use_gaussian_global_scale,+ )+++def _value_page(+ value_pages: cute.Tensor,+ value_pages_fp4: cute.Tensor,+ value_scales: cute.Tensor,+ source_tokens: cute.Tensor,+ destination_pages: cute.Tensor,+ destination_bases: cute.Tensor,+ item: Int32,+ layer: Int32,+ source_offset: Int32,+ heads: cutlass.Constexpr[int],+ sf_vec_size: cutlass.Constexpr[int],+ rest_k: cutlass.Constexpr[int],+ rule: cutlass.Constexpr,+):+ """One head of one V page, one channel across the page per thread."""+ thread, _, _ = cute.arch.thread_idx()+ head = item % heads+ source_token, page = _resolve_work(+ source_tokens, destination_pages, item // heads+ )+ if page >= 0:+ source = value_pages[source_token + source_offset, None, head, thread]+ _quantize_vector(+ _load_vector(source),+ _rebase(value_pages_fp4, destination_bases, 2, layer)[+ page, head, thread, None+ ],+ _rebase(value_scales, destination_bases, 3, layer),+ page,+ thread % 32,+ thread // 32,+ head,+ sf_vec_size,+ rest_k,+ rule,+ const_expr(False),+ )+++def _tail_token(+ key: cute.Tensor,+ value: cute.Tensor,+ key_shift: cute.Tensor,+ tail_key: cute.Tensor,+ tail_value: cute.Tensor,+ query_start_loc: cute.Tensor,+ seq_lens: cute.Tensor,+ seqused_fp4: cute.Tensor,+ row_to_slot: cute.Tensor,+ item: Int32,+ layer: Int32,+ decode_rows: Int32,+ first_row: Int32,+ heads: cutlass.Constexpr[int],+ rotate: cutlass.Constexpr[bool],+ sink_tokens: cutlass.Constexpr[int],+ tail_chunks: cutlass.Constexpr[int],+):+ """One token into the BF16 rolling page and/or sink.++ A one-token row in ``[first_row, decode_rows)`` gets one CTA for its+ token. Every later row gets ``PAGE_SIZE + sink_tokens`` candidates: one+ per rolling position past its FP4 prefix, then one per sink position, a+ sink candidate standing down where the rolling side already covers its+ token so that a token in both regions is written by one CTA, to both.+ Candidates past the row's length, before this step's tokens, or in a row+ with no slot exit. ``tail_chunks`` CTAs share a token when its groups+ outnumber one CTA's threads.+ """+ thread, _, _ = cute.arch.thread_idx()+ candidates = const_expr(PAGE_SIZE + sink_tokens)+ candidate = item // tail_chunks+ decode_count = decode_rows - first_row+ is_decode = candidate < decode_count+ later = candidate - decode_count+ row = Int32(+ cutlass.select_(+ is_decode, first_row + candidate, decode_rows + later // candidates+ )+ )+ within = later % candidates+ start = query_start_loc[row]+ seq_len = seq_lens[row]+ quantized = seqused_fp4[row]+ slot = row_to_slot[row]+ computed = seq_len - (query_start_loc[row + 1] - start)+ rolling = within < PAGE_SIZE+ position = Int32(+ cutlass.select_(+ is_decode,+ seq_len - 1,+ cutlass.select_(rolling, quantized + within, within - PAGE_SIZE),+ )+ )+ if (+ (slot >= 0)+ & (position < seq_len)+ & (position >= computed)+ & (is_decode | rolling | (position < quantized))+ ):+ shift = None+ if const_expr(key_shift is not None):+ shift = key_shift[layer, None, None]+ _write_tail_token(+ key,+ value,+ shift,+ tail_key[layer, None, None, None, None],+ tail_value[layer, None, None, None, None],+ start + position - computed,+ slot,+ position,+ position - quantized,+ (item % tail_chunks) * 128 + thread,+ heads,+ HEAD_DIM,+ rotate,+ key_shift is not None,+ sink_tokens,+ )+++def _write_kv_kernel(+ key: cute.Tensor,+ value: cute.Tensor,+ value_pages: cute.Tensor,+ key_pages_fp4: cute.Tensor,+ key_scales: cute.Tensor,+ value_pages_fp4: cute.Tensor,+ value_scales: cute.Tensor,+ source_tokens: cute.Tensor,+ destination_pages: cute.Tensor,+ query_start_loc: cute.Tensor,+ seq_lens: cute.Tensor,+ seqused_fp4: cute.Tensor,+ row_to_slot: cute.Tensor,+ tail_key: cute.Tensor,+ tail_value: cute.Tensor,+ key_shift: cute.Tensor,+ destination_bases: cute.Tensor,+ layer_index: Int32,+ decode_rows: Int32,+ first_row: Int32,+ key_end: Int32,+ value_end: Int32,+ source_layer_stride: Int32,+ heads: cutlass.Constexpr[int],+ key_sf_vec_size: cutlass.Constexpr[int],+ key_rest_k: cutlass.Constexpr[int],+ key_rule: cutlass.Constexpr,+ value_sf_vec_size: cutlass.Constexpr[int],+ value_rest_k: cutlass.Constexpr[int],+ value_rule: cutlass.Constexpr,+ use_gaussian_global_scale: cutlass.Constexpr[bool],+ rotate: cutlass.Constexpr[bool],+ sink_tokens: cutlass.Constexpr[int],+ tail_chunks: cutlass.Constexpr[int],+):+ """K pages, then V pages, then tail tokens, on one flat CTA axis.++ The roles share no data and need no synchronization between CTAs. A+ per-layer launch (``destination_bases``) takes the layer from the grid,+ a one-layer launch from ``layer_index``, which also selects that layer's+ shift and tail.+ """+ item, _, grid_layer = cute.arch.block_idx()+ layer = layer_index+ if const_expr(destination_bases is not None):+ layer = grid_layer+ source_offset = grid_layer * source_layer_stride+ if item < key_end:+ if const_expr(key_pages_fp4 is not None):+ _key_page(+ key,+ key_shift,+ key_pages_fp4,+ key_scales,+ source_tokens,+ destination_pages,+ destination_bases,+ item,+ layer,+ source_offset,+ heads,+ key_sf_vec_size,+ key_rest_k,+ key_rule,+ use_gaussian_global_scale,+ rotate,+ )+ elif item < value_end:+ if const_expr(value_pages_fp4 is not None):+ _value_page(+ value_pages,+ value_pages_fp4,+ value_scales,+ source_tokens,+ destination_pages,+ destination_bases,+ item - key_end,+ layer,+ source_offset,+ heads,+ value_sf_vec_size,+ value_rest_k,+ value_rule,+ )+ else:+ if const_expr(tail_key is not None):+ _tail_token(+ key,+ value,+ key_shift,+ tail_key,+ tail_value,+ query_start_loc,+ seq_lens,+ seqused_fp4,+ row_to_slot,+ item - value_end,+ layer,+ decode_rows,+ first_row,+ heads,+ rotate,+ sink_tokens,+ tail_chunks,+ )+++class _KVWrite:+ """The compile-time half of :class:`KVCacheWriter`: one launch configuration."""++ def __init__(+ self,+ *,+ heads: int,+ write_key: bool,+ write_value: bool,+ write_tail: bool,+ key_scale_mode: str,+ value_scale_mode: str,+ scale_geometry: str,+ rotate: bool,+ sink_tokens: int,+ ) -> None:+ self.heads = heads+ self.write_key = write_key+ self.write_value = write_value+ self.write_tail = write_tail+ self.key_sf_vec_size = sf_vec_size(key_scale_mode)+ self.key_rest_k = scale_atoms(HEAD_DIM, key_scale_mode)+ self.key_rule = scale_rule(key_scale_mode)+ self.value_sf_vec_size = sf_vec_size(value_scale_mode)+ self.value_rest_k = scale_atoms(PAGE_SIZE, value_scale_mode)+ self.value_rule = scale_rule(value_scale_mode)+ self.use_gaussian_global_scale = scale_geometry == "gaussian_global"+ self.rotate = rotate+ self.sink_tokens = sink_tokens+ # A token's K groups padded to whole warps, then its V groups.+ groups = heads * (HEAD_DIM // SF_VEC_SIZE)+ self.tail_chunks = ((groups + 31) // 32 * 32 + groups + 127) // 128+ self.smem = 0++ @cute.jit+ def __call__(+ self,+ key,+ value,+ key_pages_fp4,+ key_scales,+ value_pages_fp4,+ value_scales,+ source_tokens: cute.Tensor,+ destination_pages: cute.Tensor,+ query_start_loc,+ seq_lens,+ seqused_fp4,+ row_to_slot,+ tail_key,+ tail_value,+ key_shift,+ destination_bases,+ layer_index: Int32,+ decode_rows: Int32,+ first_row: Int32,+ stream,+ ):+ layers = Int32(1)+ if const_expr(destination_bases is not None):+ layers = destination_bases.shape[1]+ page_items = self.heads * source_tokens.shape[0]+ key_end = Int32(0)+ if const_expr(self.write_key):+ key_end = page_items+ value_end = key_end+ if const_expr(self.write_value):+ value_end = key_end + page_items+ tail_items = Int32(0)+ if const_expr(self.write_tail):+ tail_items = (+ (decode_rows - first_row)+ + (seq_lens.shape[0] - decode_rows) * (PAGE_SIZE + self.sink_tokens)+ ) * self.tail_chunks+ # One destination per layer means one source per layer too, stacked in+ # that order along the token axis.+ tokens = key+ if const_expr(key is None):+ tokens = value+ source_layer_stride = tokens.shape[0] // layers+ # V reads one channel down a page: a (page, token, head, dim) view whose+ # page axis has a one-token stride, so the leading index is a token+ # offset and a work item can start anywhere. The axes overlap, which is+ # harmless for a read.+ value_pages = None+ if const_expr(value is not None):+ value_pages = cute.make_tensor(+ value.iterator,+ cute.make_layout(+ (source_tokens.shape[0], PAGE_SIZE, self.heads, HEAD_DIM),+ stride=(value.stride[0], value.stride[0], HEAD_DIM, 1),+ ),+ )+ _write_kv_kernel(+ key,+ value,+ value_pages,+ key_pages_fp4,+ _as_bytes(key_scales),+ value_pages_fp4,+ _as_bytes(value_scales),+ source_tokens,+ destination_pages,+ query_start_loc,+ seq_lens,+ seqused_fp4,+ row_to_slot,+ tail_key,+ tail_value,+ key_shift,+ destination_bases,+ layer_index,+ decode_rows,+ first_row,+ key_end,+ value_end,+ source_layer_stride,+ const_expr(self.heads),+ const_expr(self.key_sf_vec_size),+ const_expr(self.key_rest_k),+ const_expr(self.key_rule),+ const_expr(self.value_sf_vec_size),+ const_expr(self.value_rest_k),+ const_expr(self.value_rule),+ const_expr(self.use_gaussian_global_scale),+ const_expr(self.rotate),+ const_expr(self.sink_tokens),+ const_expr(self.tail_chunks),+ # The per-layer form is promotion's; a profile tells it from the+ # write path's launches by name.+ _name_prefix="seal" if destination_bases is not None else None,+ ).launch(+ grid=(value_end + tail_items, 1, layers),+ block=(128, 1, 1),+ smem=self.smem,+ stream=stream,+ )+++def _as_bytes(scales):+ if const_expr(scales is None):+ return None+ return cute.make_tensor(+ cute.recast_ptr(scales.iterator, dtype=cutlass.Uint8), scales.layout+ )+++_SCALE_TYPES = {+ torch.uint8: cutlass.Uint8,+ torch.float8_e4m3fn: cutlass.Float8E4M3FN,+ torch.float8_e8m0fnu: cutlass.Float8E8M0FNU,+}+_compiled_writers: dict[tuple, object] = {}+++def _compile_writer(+ *,+ heads: int,+ key_scales_dtype: torch.dtype | None,+ value_scales_dtype: torch.dtype | None,+ key_scale_mode: str,+ value_scale_mode: str,+ scale_geometry: str,+ write_tail: bool,+ shift_dtype: torch.dtype | None,+ rotate: bool,+ per_layer: bool,+ sink_tokens: int,+):+ """The compiled launch for one configuration; every page interior static.++ TVM-FFI checks each call's tensors against these declarations (dtype,+ rank, device, alignment, every static extent and stride, and equality of+ shared symbols), which is the per-call validation: only token pitches,+ block pitches, and counts are free. Shapes are 32-bit, pitches 64-bit,+ since a block index times a block pitch passes 2^31 at 14563 blocks.+ """+ cache_key = (+ torch.cuda.current_device(),+ heads,+ key_scales_dtype,+ value_scales_dtype,+ key_scale_mode,+ value_scale_mode,+ scale_geometry,+ write_tail,+ shift_dtype,+ rotate,+ per_layer,+ sink_tokens,+ )+ compiled = _compiled_writers.get(cache_key)+ if compiled is not None:+ return compiled++ def fake(dtype, shape, stride, align):+ return make_fake_tensor(dtype, shape, stride, assumed_align=align)++ sym32, sym64 = cute.sym_int32, cute.sym_int64+ write_key = key_scales_dtype is not None+ write_value = value_scales_dtype is not None++ def token_major():+ return fake(+ cutlass.BFloat16,+ (sym32(), heads, HEAD_DIM),+ (sym64(divisibility=8), HEAD_DIM, 1),+ 16,+ )++ blocks, layers, work = sym32(), sym32(), sym32()++ def regions(packed_shape, scales_dtype, rest_k):+ packed = fake(+ cutlass.Uint8,+ (blocks,) + packed_shape,+ (sym64(divisibility=16),) + _dense_strides(packed_shape),+ 16,+ )+ scales = fake(+ _SCALE_TYPES[scales_dtype],+ (blocks, 32, 4, 1, 4, rest_k, heads),+ (sym64(divisibility=16), 16, 4, rest_k * 512, 1, 512, rest_k * 512),+ 16,+ )+ return packed, scales++ key = token_major() if write_key or write_tail else None+ value = token_major() if write_value or write_tail else None+ key_regions = (None, None)+ if write_key:+ key_regions = regions(+ (PAGE_SIZE, heads, HEAD_DIM // 2),+ key_scales_dtype,+ scale_atoms(HEAD_DIM, key_scale_mode),+ )+ value_regions = (None, None)+ if write_value:+ value_regions = regions(+ (heads, HEAD_DIM, PAGE_SIZE // 2),+ value_scales_dtype,+ scale_atoms(PAGE_SIZE, value_scale_mode),+ )+ rows, row_vectors, tails = sym32(), (None,) * 4, (None, None)+ if write_tail:+ row_vectors = (+ fake(Int32, (sym32(),), (1,), 4),+ *(fake(Int32, (rows,), (1,), 4) for _ in range(3)),+ )+ tail_shape = (layers, sym32(), PAGE_SIZE + sink_tokens, heads, HEAD_DIM)+ tail_strides = (sym64(), sym64()) + _dense_strides(tail_shape[2:])+ tails = (+ fake(cutlass.BFloat16, tail_shape, tail_strides, 16),+ fake(cutlass.BFloat16, tail_shape, tail_strides, 16),+ )+ shift = None+ if shift_dtype is not None:+ shift = fake(+ {torch.float32: cutlass.Float32, torch.bfloat16: cutlass.BFloat16}[+ shift_dtype+ ],+ (layers, heads, HEAD_DIM),+ (heads * HEAD_DIM, HEAD_DIM, 1),+ 16,+ )+ bases = (+ fake(cutlass.Int64, (4, layers), (sym64(), 1), 8) if per_layer else None+ )+ launch = _KVWrite(+ heads=heads,+ write_key=write_key,+ write_value=write_value,+ write_tail=write_tail,+ key_scale_mode=key_scale_mode,+ value_scale_mode=value_scale_mode,+ scale_geometry=scale_geometry,+ rotate=rotate,+ sink_tokens=sink_tokens,+ )+ compiled = cute.compile(+ launch,+ key,+ value,+ *key_regions,+ *value_regions,+ fake(Int32, (work,), (1,), 4),+ fake(Int32, (work,), (1,), 4),+ *row_vectors,+ *tails,+ shift,+ bases,+ Int32(0),+ Int32(0),+ Int32(0),+ make_fake_stream(use_tvm_ffi_env_stream=True),+ options="--enable-tvm-ffi",+ )+ _compiled_writers[cache_key] = compiled+ return compiled+++def _validate_destination(+ pages_fp4: torch.Tensor,+ scales: torch.Tensor,+ *,+ heads: int,+ inner_shape: tuple[int, ...],+ rest_k: int,+) -> None:+ """Reject a destination whose page interior is not the quantizer's layout.++ Only the page pitch is free. Everything inside a page is the layout the+ decode kernel reads back, so a mismatch there is silent corruption.+ """+ if (+ pages_fp4.dtype is not torch.uint8+ or not pages_fp4.is_cuda+ or tuple(pages_fp4.shape[1:]) != inner_shape+ or pages_fp4.stride()[1:] != _dense_strides(inner_shape)+ or pages_fp4.stride(0) % 16+ ):+ raise ValueError(+ f"packed destination must be uint8 CUDA pages of {inner_shape}, "+ "dense within a page, 16-byte page pitch"+ )+ if (+ scales.dtype not in _SCALE_TYPES+ or scales.device != pages_fp4.device+ or tuple(scales.shape[1:]) != (32, 4, 1, 4, rest_k, heads)+ or scales.stride()[1:] != (16, 4, rest_k * 512, 1, 512, rest_k * 512)+ or scales.stride(0) % 16+ ):+ raise ValueError(+ "scale destination must use the quantizer's own scale layout"+ )+ if pages_fp4.shape[0] != scales.shape[0]:+ raise ValueError("packed and scale destinations must have equal pages")+++class KVCacheWriter:+ """K/V into FP4 pages and the BF16 tail, one launch per call.++ Constructed once with everything that does not change between steps, and+ all of it is checked here: the scale modes, geometry, the tail buffers+ ``[layers, slots, 128 + sink_tokens, heads, 128]``, the shift+ ``[layers, heads, 128]`` (FP32, or BF16, already in the rotated basis),+ and the ``[4, layers]`` base-address table of a per-layer writer. A call+ passes only what changes: the source K/V, a layer's four regions (as+ :meth:`bind_regions` returned them, or as tensors, which TVM-FFI then+ converts on every call), the step's work table and row+ metadata, the layer index, the decode row count and the first row whose+ tail token this writes. TVM-FFI checks each call's tensors against the+ layouts compiled in.++ The roles present are those given: K pages with K regions, V pages with V+ regions, tail tokens with tail buffers. A per-layer writer (one with+ ``destination_bases``) quantizes the same work items for every layer at+ once from a source holding the layers back to back, each into the+ destination its row of the table names; it writes no tail.++ ``rotate_key`` and ``key_shift`` put raw K into the cache's basis on the+ way in, for the pages and the tail alike, with the arithmetic of+ :func:`_cache_basis`. A work item reads the 128 tokens from its source+ token and writes its destination page; a negative page skips it, which is+ how a fixed launch shape covers a varying batch.+ """++ def __init__(+ self,+ *,+ regions: tuple,+ key_scale_mode: str = DEFAULT_SCALE_MODE,+ value_scale_mode: str = DEFAULT_SCALE_MODE,+ key_scale_geometry: str = "local",+ rotate_key: bool = False,+ key_shift: torch.Tensor | None = None,+ tail_key: torch.Tensor | None = None,+ tail_value: torch.Tensor | None = None,+ sink_tokens: int = 0,+ destination_bases: torch.Tensor | None = None,+ ) -> None:+ key_packed, key_scales, value_packed, value_scales = regions+ any_packed = key_packed if key_packed is not None else value_packed+ if any_packed is None:+ raise ValueError("a writer needs K or V regions")+ heads = any_packed.shape[2 if key_packed is not None else 1]+ self.heads = heads+ self.key_scale_mode = key_scale_mode+ self.value_scale_mode = value_scale_mode+ rules = (scale_rule(key_scale_mode), scale_rule(value_scale_mode))+ if any(isinstance(rule, str) for rule in rules) and key_scale_geometry != "local":+ raise ValueError(+ "MXFP4 pages support local scales only"+ )+ if key_scale_geometry not in KEY_SCALE_GEOMETRIES:+ raise ValueError(+ f"unknown key_scale_geometry {key_scale_geometry!r}, expected "+ f"one of {list(KEY_SCALE_GEOMETRIES)}"+ )+ if key_packed is None and (key_shift is not None or rotate_key):+ raise ValueError("the K transform needs K regions")+ if (tail_key is None) != (tail_value is None):+ raise ValueError("tail_key and tail_value come together")+ if tail_key is not None and destination_bases is not None:+ raise ValueError("a per-layer writer writes no tail")+ device = any_packed.device+ layers = None++ def layer_axis(tensor: torch.Tensor, name: str, inner: tuple, dtypes):+ nonlocal layers+ if (+ tensor.dtype not in dtypes+ or tensor.device != device+ or not tensor.is_contiguous()+ or tuple(tensor.shape[1:]) != inner+ ):+ raise ValueError(+ f"{name} must be contiguous {' or '.join(map(str, dtypes))} "+ f"[layers, *{inner}] on {device}"+ )+ if layers is not None and tensor.shape[0] != layers:+ raise ValueError(+ f"{name} has {tensor.shape[0]} layers, not {layers}"+ )+ layers = tensor.shape[0]++ if destination_bases is not None:+ if (+ destination_bases.dtype is not torch.int64+ or destination_bases.device != device+ or not destination_bases.is_contiguous()+ or destination_bases.ndim != 2+ or destination_bases.shape[0] != 4+ ):+ raise ValueError(+ "destination_bases must be a contiguous int64 [4, layers] "+ "table of K packed, K scale, V packed and V scale addresses"+ )+ layers = destination_bases.shape[1]+ if tail_key is not None:+ row = (PAGE_SIZE + sink_tokens, heads, HEAD_DIM)+ for name, tensor in (("tail_key", tail_key), ("tail_value", tail_value)):+ if (+ tensor.dtype is not torch.bfloat16+ or tensor.device != device+ or not tensor.is_contiguous()+ or tensor.ndim != 5+ or tuple(tensor.shape[2:]) != row+ or tensor.shape != tail_key.shape+ ):+ raise ValueError(+ f"{name} must be contiguous BF16 [layers, slots, "+ f"{PAGE_SIZE + sink_tokens}, {heads}, {HEAD_DIM}] on {device}"+ )+ layers = tail_key.shape[0]+ if key_shift is not None:+ layer_axis(+ key_shift,+ "key_shift",+ (heads, HEAD_DIM),+ (torch.float32, torch.bfloat16),+ )+ self._regions_dtypes = (+ None if key_scales is None else key_scales.dtype,+ None if value_scales is None else value_scales.dtype,+ )+ self.bind_regions(*regions)+ self._compiled = _compile_writer(+ heads=heads,+ key_scales_dtype=self._regions_dtypes[0],+ value_scales_dtype=self._regions_dtypes[1],+ key_scale_mode=key_scale_mode,+ value_scale_mode=value_scale_mode,+ scale_geometry=key_scale_geometry,+ write_tail=tail_key is not None,+ shift_dtype=None if key_shift is None else key_shift.dtype,+ rotate=bool(rotate_key),+ per_layer=destination_bases is not None,+ sink_tokens=sink_tokens,+ )+ self.writes_tail = tail_key is not None+ self.layers = layers if destination_bases is not None else None+ # Converted once: a tvm_ffi.Tensor crosses into the compiled call as a+ # pointer, where a torch tensor is converted on every call.+ self._constants = tuple(+ None if tensor is None else tvm_ffi.from_dlpack(tensor)+ for tensor in (+ tail_key,+ tail_value,+ key_shift,+ destination_bases,+ )+ )++ def bind_regions(+ self,+ key_pages_fp4: torch.Tensor | None,+ key_scales: torch.Tensor | None,+ value_pages_fp4: torch.Tensor | None,+ value_scales: torch.Tensor | None,+ ) -> tuple:+ """Check one layer's four regions once and keep them call-ready."""+ dtypes = (+ None if key_scales is None else key_scales.dtype,+ None if value_scales is None else value_scales.dtype,+ )+ if dtypes != self._regions_dtypes:+ raise ValueError(+ f"regions carry scale dtypes {dtypes}, this writer was built for "+ f"{self._regions_dtypes}"+ )+ if key_pages_fp4 is not None:+ _validate_destination(+ key_pages_fp4,+ key_scales,+ heads=self.heads,+ inner_shape=(PAGE_SIZE, self.heads, HEAD_DIM // 2),+ rest_k=scale_atoms(HEAD_DIM, self.key_scale_mode),+ )+ if value_pages_fp4 is not None:+ _validate_destination(+ value_pages_fp4,+ value_scales,+ heads=self.heads,+ inner_shape=(self.heads, HEAD_DIM, PAGE_SIZE // 2),+ rest_k=scale_atoms(PAGE_SIZE, self.value_scale_mode),+ )+ return tuple(+ None if region is None else tvm_ffi.from_dlpack(region)+ for region in (key_pages_fp4, key_scales, value_pages_fp4, value_scales)+ )++ def __call__(+ self,+ key: torch.Tensor | None,+ value: torch.Tensor | None,+ regions: tuple,+ source_tokens: torch.Tensor,+ destination_pages: torch.Tensor,+ query_start_loc: torch.Tensor | None = None,+ seq_lens: torch.Tensor | None = None,+ seqused_fp4: torch.Tensor | None = None,+ row_to_slot: torch.Tensor | None = None,+ layer: int = 0,+ decode_rows: int = 0,+ first_row: int = 0,+ ) -> None:+ """Write this call's pages and tail tokens; launch nothing if there are none.++ ``key`` and ``value`` are ``[tokens, heads, 128]`` BF16, dense within a+ token; a per-layer writer's holds the layers back to back. Rows+ ``[0, decode_rows)`` of the row metadata are one-token rows; rows+ before ``first_row`` get no tail write.+ """+ if not destination_pages.shape[0] and (+ not self.writes_tail or seq_lens.shape[0] == first_row+ ):+ return+ if self.layers is not None:+ tokens = (key if key is not None else value).shape[0]+ if tokens % self.layers:+ # Rounding the per-layer stride down would shift every layer+ # but the first by a few tokens: a plausible-looking cache,+ # quietly out of step.+ raise ValueError(+ f"a source of {tokens} tokens does not divide into "+ f"{self.layers} layers"+ )+ tail_key, tail_value, key_shift, bases = self._constants+ self._compiled(+ key,+ value,+ *regions,+ source_tokens,+ destination_pages,+ query_start_loc,+ seq_lens,+ seqused_fp4,+ row_to_slot,+ tail_key,+ tail_value,+ key_shift,+ bases,+ layer,+ decode_rows,+ first_row,+ )+++def scale_atoms(extent: int, scale_mode: str) -> int:+ """How many four-group scale atoms cover ``extent`` values under ``scale_mode``."""+ return extent // (GROUPS_PER_ATOM * sf_vec_size(scale_mode))+++def scale_dtype(scale_mode: str) -> torch.dtype:+ """E4M3 for an NVFP4 mode, E8M0 for an MX mode."""+ if scale_format(scale_mode) == "mxfp4":+ return torch.float8_e8m0fnu+ return torch.float8_e4m3fn+++def _validate_pages(pages: torch.Tensor, name: str) -> tuple[int, int]:+ if (+ pages.dtype is not torch.bfloat16+ or not pages.is_cuda+ or not pages.is_contiguous()+ or pages.ndim != 4+ ):+ raise ValueError(+ f"{name} must be contiguous BF16 CUDA with shape "+ "[pages, 128, heads, 128]"+ )+ page_count, page_size, heads, head_dim = pages.shape+ if page_count < 1 or page_size != PAGE_SIZE or head_dim != HEAD_DIM:+ raise ValueError(+ f"{name} must have shape [pages, 128, heads, 128]"+ )+ return page_count, heads+++def _allocate_scales(+ page_count: int,+ heads: int,+ rest_m: int,+ rest_k: int,+ device: torch.device,+) -> tuple[torch.Tensor, torch.Tensor]:+ atom_bytes = 512+ m_block_stride = rest_k * atom_bytes+ head_stride = rest_m * m_block_stride+ page_stride = heads * head_stride+ shape = (page_count, 32, 4, rest_m, 4, rest_k, heads)+ strides = (+ page_stride,+ 16,+ 4,+ m_block_stride,+ 1,+ atom_bytes,+ head_stride,+ )+ storage = torch.empty(+ page_count * page_stride,+ dtype=torch.uint8,+ device=device,+ )+ return storage, storage.as_strided(shape, strides)+++def _dense_strides(shape: tuple) -> tuple:+ """Row-major strides. The leading extent, which may be symbolic, is not used."""+ strides = [1]+ for extent in reversed(shape[1:]):+ strides.append(strides[-1] * extent)+ return tuple(reversed(strides))+++def _identity_work(page_count: int, device) -> tuple[torch.Tensor, torch.Tensor]:+ """Work item ``w`` reads source page ``w`` and writes destination page ``w``."""+ pages = torch.arange(page_count, dtype=torch.int32, device=device)+ return pages * PAGE_SIZE, pages+++def quantize_key_pages(+ key_pages: torch.Tensor,+ scale_mode: str = DEFAULT_SCALE_MODE,+ scale_geometry: str = "local",+) -> tuple[torch.Tensor, torch.Tensor]:+ """Quantize BF16 K pages into freshly allocated E2M1 and their scales."""+ page_count, heads = _validate_pages(key_pages, "key_pages")+ rest_k = scale_atoms(HEAD_DIM, scale_mode)+ key_pages_fp4 = torch.empty(+ page_count,+ PAGE_SIZE,+ heads,+ HEAD_DIM // 2,+ dtype=torch.uint8,+ device=key_pages.device,+ )+ _, key_scales = _allocate_scales(+ page_count,+ heads,+ 1,+ rest_k,+ key_pages.device,+ )+ regions = (key_pages_fp4, key_scales, None, None)+ writer = KVCacheWriter(+ regions=regions,+ key_scale_mode=scale_mode,+ key_scale_geometry=scale_geometry,+ )+ writer(+ key_pages.reshape(-1, heads, HEAD_DIM),+ None,+ regions,+ *_identity_work(page_count, key_pages.device),+ )+ return key_pages_fp4, key_scales.view(scale_dtype(scale_mode))+++def quantize_value_pages(+ value_pages: torch.Tensor,+ scale_mode: str = DEFAULT_SCALE_MODE,+) -> tuple[torch.Tensor, torch.Tensor]:+ """Quantize BF16 V pages into freshly allocated E2M1 and their scales."""+ page_count, heads = _validate_pages(value_pages, "value_pages")+ rest_m = HEAD_DIM // 128+ page_rest_k = scale_atoms(PAGE_SIZE, scale_mode)+ value_pages_fp4 = torch.empty(+ page_count,+ heads,+ HEAD_DIM,+ PAGE_SIZE // 2,+ dtype=torch.uint8,+ device=value_pages.device,+ )+ _, value_scales = _allocate_scales(+ page_count,+ heads,+ rest_m,+ page_rest_k,+ value_pages.device,+ )+ regions = (None, None, value_pages_fp4, value_scales)+ writer = KVCacheWriter(regions=regions, value_scale_mode=scale_mode)+ writer(+ None,+ value_pages.reshape(-1, heads, HEAD_DIM),+ regions,+ *_identity_work(page_count, value_pages.device),+ )+ return value_pages_fp4, value_scales.view(scale_dtype(scale_mode))diff --git a/src/nvfp4_decode_kernel/quantize_q_kernel.py b/src/nvfp4_decode_kernel/quantize_q_kernel.pynew file mode 100644index 0000000..4a4e4a1--- /dev/null+++ b/src/nvfp4_decode_kernel/quantize_q_kernel.py@@ -0,0 +1,1110 @@+"""CuTeDSL decode-query quantization."""++import math+from typing import Optional, Tuple++import cuda.bindings.driver as cuda+import torch++import cutlass+import cutlass.cute as cute+from cutlass import Float32, Int32, const_expr+from cutlass._mlir.dialects import llvm+from cutlass.cute.runtime import make_ptr+from cutlass.cutlass_dsl import T++from ._block_scale import (+ GROUPS_PER_ATOM,+ _pack_e4m3,+ _float_bits,+ _mx_round_trip,+ group_residual,+ ocp_exponent,+ power_of_two,+ round_up_exponent,+ select_group_scale,+)+from .interface import DecodeTailWrite+from .scale_mode import DEFAULT_SCALE_MODE, scale_rule, sf_vec_size+++PAGE_SIZE = 128+SF_VEC_SIZE = 16+THREADS = 128+QUERY_ROTATIONS = ("none", "hadamard")+++def _group_slice(vector: cute.Tensor, group, width: cutlass.Constexpr[int]):+ """One group's contiguous run of ``width`` elements of ``vector``.++ The host checks that every head vector starts on a 16-byte boundary, so a+ group of sixteen BF16 values or eight packed FP4 bytes is aligned to its own+ size and each copy is one or two vector accesses.+ """+ alignment = const_expr(min(16, width * vector.element_type.width // 8))+ return cute.make_tensor(+ (vector.iterator + group * width).align(alignment),+ cute.make_layout(width),+ )+++def _mul_rn(a: Float32, b: Float32, *, loc=None, ip=None) -> Float32:+ """``a * b`` rounded on its own, never contracted into a later add."""+ product = llvm.inline_asm(+ T.f32(),+ [+ Float32(a).ir_value(loc=loc, ip=ip),+ Float32(b).ir_value(loc=loc, ip=ip),+ ],+ "mul.rn.f32 $0, $1, $2;",+ "=f,f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ return Float32(product)+++def _sub_rn(a: Float32, b: Float32, *, loc=None, ip=None) -> Float32:+ difference = llvm.inline_asm(+ T.f32(),+ [+ Float32(a).ir_value(loc=loc, ip=ip),+ Float32(b).ir_value(loc=loc, ip=ip),+ ],+ "sub.rn.f32 $0, $1, $2;",+ "=f,f,f",+ has_side_effects=False,+ is_align_stack=False,+ asm_dialect=llvm.AsmDialect.AD_ATT,+ )+ return Float32(difference)+++def _hadamard(+ values: cute.Tensor,+ group: Int32,+ lanes_per_head: cutlass.Constexpr[int],+ head_dim: cutlass.Constexpr[int],+):+ """Multiply one head by the normalized Sylvester Hadamard, in place.++ The ``±1`` butterflies of :func:`_butterflies`, then the single+ ``1 / sqrt(head_dim)`` multiply.+ """+ _butterflies(values, group, lanes_per_head)+ inverse = Float32(1.0 / math.sqrt(head_dim))+ for j in cutlass.range_constexpr(SF_VEC_SIZE):+ values[j] = Float32(values[j]) * inverse+++def _butterflies(+ values: cute.Tensor,+ group: Int32,+ lanes_per_head: cutlass.Constexpr[int],+):+ """One head times the unnormalized Sylvester Hadamard, in place.++ Channel ``SF_VEC_SIZE * group + j`` is register ``j`` of the head's lane+ ``group``. The low four index bits are butterflied in registers and the+ rest across the head's aligned lanes, lowest bit first, which is the order+ the reference reproduces. Each stage maps ``(a, c)`` at indices with the+ bit clear and set to ``(a + c, a - c)`` in FP32.+ """+ for bit in cutlass.range_constexpr(4):+ stride = 1 << bit+ for j in cutlass.range_constexpr(SF_VEC_SIZE):+ if const_expr((j & stride) == 0):+ low = values[j]+ high = values[j + stride]+ values[j] = low + high+ values[j + stride] = low - high+ for bit in cutlass.range_constexpr(int(math.log2(lanes_per_head))):+ mask = 1 << bit+ upper = (group & mask) != 0+ for j in cutlass.range_constexpr(SF_VEC_SIZE):+ own = Float32(values[j])+ partner = cute.arch.shuffle_sync_bfly(own, offset=mask)+ values[j] = Float32(+ cutlass.select_(upper, partner - own, own + partner)+ )+++def _write_tail_token(+ key: cute.Tensor,+ value: cute.Tensor,+ key_shift: cute.Tensor,+ tail_key: cute.Tensor,+ tail_value: cute.Tensor,+ token: Int32,+ slot: Int32,+ position: Int32,+ offset: Int32,+ item: Int32,+ heads_kv: cutlass.Constexpr[int],+ head_dim: cutlass.Constexpr[int],+ rotate: cutlass.Constexpr[bool],+ shift: cutlass.Constexpr[bool],+ sink_tokens: cutlass.Constexpr[int],+):+ """One 16-element group of one token's new K or V, into the tail.++ ``token`` indexes ``key`` and ``value``; ``position`` is its place in the+ sequence and ``offset`` its place in the rolling page (negative: not in+ it). It is stored at ``offset`` when that is not negative and at+ ``PAGE_SIZE + position`` in the sink below ``sink_tokens``, both from this+ one call. K is rotated by FP32 butterflies, scaled by one separately+ rounded multiply, shifted by one separately rounded subtract of the FP32+ (or exactly widened BF16) shift, and rounded to BF16 once: the arithmetic+ of the K page writer, ``quantize_kv_kernel._cache_basis``.++ K groups fill whole warps before V starts, so the butterfly shuffles never+ see a lane that took the V side; K lanes past the last head clamp their+ head and only skip the store.+ """+ groups_per_head = const_expr(head_dim // SF_VEC_SIZE)+ items = const_expr(heads_kv * groups_per_head)+ key_span = const_expr((items + 31) // 32 * 32)++ if item < key_span:+ active = item < items+ head = Int32(+ cutlass.select_(active, item // groups_per_head, heads_kv - 1)+ )+ group = item % groups_per_head+ loaded = cute.make_rmem_tensor(+ cute.make_layout(SF_VEC_SIZE), cutlass.BFloat16+ )+ cute.autovec_copy(+ _group_slice(key[token, head, None], group, SF_VEC_SIZE), loaded+ )+ if const_expr(rotate or shift):+ values = cute.make_rmem_tensor(+ cute.make_layout(SF_VEC_SIZE), Float32+ )+ values.store(loaded.load().to(Float32))+ if const_expr(rotate):+ _butterflies(values, group, groups_per_head)+ inverse = Float32(1.0 / math.sqrt(head_dim))+ for j in cutlass.range_constexpr(SF_VEC_SIZE):+ values[j] = _mul_rn(values[j], inverse)+ if const_expr(shift):+ centre = cute.make_rmem_tensor(+ cute.make_layout(SF_VEC_SIZE), key_shift.element_type+ )+ cute.autovec_copy(+ _group_slice(key_shift[head, None], group, SF_VEC_SIZE),+ centre,+ )+ for j in cutlass.range_constexpr(SF_VEC_SIZE):+ values[j] = _sub_rn(values[j], Float32(centre[j]))+ loaded.store(values.load().to(cutlass.BFloat16))+ if active:+ if slot >= 0:+ if offset >= 0:+ cute.autovec_copy(+ loaded,+ _group_slice(+ tail_key[slot, offset, head, None],+ group,+ SF_VEC_SIZE,+ ),+ )+ if position < sink_tokens:+ cute.autovec_copy(+ loaded,+ _group_slice(+ tail_key[slot, PAGE_SIZE + position, head, None],+ group,+ SF_VEC_SIZE,+ ),+ )+ else:+ v_index = item - key_span+ if v_index < items:+ v_head = v_index // groups_per_head+ v_group = v_index % groups_per_head+ v_loaded = cute.make_rmem_tensor(+ cute.make_layout(SF_VEC_SIZE), cutlass.BFloat16+ )+ cute.autovec_copy(+ _group_slice(value[token, v_head, None], v_group, SF_VEC_SIZE),+ v_loaded,+ )+ if slot >= 0:+ if offset >= 0:+ cute.autovec_copy(+ v_loaded,+ _group_slice(+ tail_value[slot, offset, v_head, None],+ v_group,+ SF_VEC_SIZE,+ ),+ )+ if position < sink_tokens:+ cute.autovec_copy(+ v_loaded,+ _group_slice(+ tail_value[slot, PAGE_SIZE + position, v_head, None],+ v_group,+ SF_VEC_SIZE,+ ),+ )+++def _scale_word(scale: Float32, slot: Int32) -> Int32:+ """The atom's four E4M3 scales as one word, gathered from its four lanes.++ Byte ``slot`` of the word belongs to group ``slot`` of the atom, and the+ atom's groups sit in four consecutive lanes, so two butterfly ORs give+ every one of them the whole word; lane ``slot == 0`` stores it.+ """+ byte = _pack_e4m3(scale, scale, scale, scale) & 0xFF+ word = Int32(byte << (slot * 8))+ word = word | cute.arch.shuffle_sync_bfly(word, offset=1)+ word = word | cute.arch.shuffle_sync_bfly(word, offset=2)+ return word+++def _store_term(+ scale: Float32,+ low: Int32,+ high: Int32,+ query_fp4: cute.Tensor,+ query_scales: cute.Tensor,+ row: Int32,+ head: Int32,+ group: Int32,+ active,+ qhead_per_kvhead: cutlass.Constexpr[int],+ is_mxfp4: cutlass.Constexpr[bool] = False,+):+ words = cute.make_rmem_tensor(cute.make_layout(2), Int32)+ words[0] = low+ words[1] = high+ if const_expr(is_mxfp4):+ # Two lanes share one scale; eight consecutive lanes make an atom.+ # The FP32 exponent bits already encode E8M0, including byte zero+ # for the single subnormal scale 2**-127.+ slot = group % (2 * GROUPS_PER_ATOM)+ byte = (_float_bits(scale) >> 23) & 0xFF+ word = Int32(byte << ((slot // 2) * 8))+ for bit in cutlass.range_constexpr(3):+ word = word | cute.arch.shuffle_sync_bfly(word, offset=1 << bit)+ scale_atom = group // (2 * GROUPS_PER_ATOM)+ else:+ slot = group % GROUPS_PER_ATOM+ word = _scale_word(scale, slot)+ scale_atom = group // GROUPS_PER_ATOM+ if active:+ destination = query_fp4[row, 0, head, None]+ cute.autovec_copy(+ words,+ cute.make_tensor(+ cute.recast_ptr(+ _group_slice(destination, group, SF_VEC_SIZE // 2).iterator,+ dtype=Int32,+ ),+ cute.make_layout(2),+ ),+ )+ if slot == 0:+ packed_row = head % qhead_per_kvhead+ scale_tile = query_scales[+ packed_row % 32,+ packed_row // 32,+ 0,+ None,+ scale_atom,+ head // qhead_per_kvhead,+ row,+ ]+ scale_word = cute.make_tensor(+ cute.recast_ptr(scale_tile.iterator, dtype=Int32),+ cute.make_layout(1),+ )+ scale_word[0] = word+++def _select_mx_query_group(values: cute.Tensor, rule: cutlass.Constexpr[str]):+ """Two adjacent lanes choose one exponent, each holding sixteen values."""+ maximum = Float32(0.0)+ for i in cutlass.range_constexpr(SF_VEC_SIZE):+ value = Float32(values[i])+ maximum = cute.arch.fmax(maximum, cute.arch.fmax(value, -value))+ maximum = cute.arch.fmax(+ maximum, cute.arch.shuffle_sync_bfly(maximum, offset=1)+ )+ if const_expr(rule == "round_up"):+ exponent = round_up_exponent(maximum)+ else:+ exponent = ocp_exponent(maximum)+ words, error = _mx_round_trip(values, Int32(0), exponent, SF_VEC_SIZE)+ if const_expr(rule == "search"):+ error = error + cute.arch.shuffle_sync_bfly(error, offset=1)+ rival = Int32(cutlass.select_(exponent < 127, exponent + 1, exponent))+ other, other_error = _mx_round_trip(values, Int32(0), rival, SF_VEC_SIZE)+ other_error = other_error + cute.arch.shuffle_sync_bfly(other_error, offset=1)+ take = other_error < error+ exponent = Int32(cutlass.select_(take, rival, exponent))+ for word in cutlass.range_constexpr(2):+ words[word] = Int32(cutlass.select_(take, other[word], words[word]))+ return power_of_two(exponent), words[0], words[1]+++def _quantize_query_kernel(+ query: cute.Tensor,+ row_indices: cute.Tensor,+ query_fp4: cute.Tensor,+ query_scales: cute.Tensor,+ query_fp4_second: cute.Tensor,+ query_scales_second: cute.Tensor,+ query_padded: cute.Tensor,+ key: cute.Tensor,+ value: cute.Tensor,+ key_shift: cute.Tensor,+ tail_key: cute.Tensor,+ tail_value: cute.Tensor,+ row_to_slot: cute.Tensor,+ seq_lens: cute.Tensor,+ seqused_fp4: cute.Tensor,+ heads: cutlass.Constexpr[int],+ rest_k: cutlass.Constexpr[int],+ use_row_indices: cutlass.Constexpr[bool],+ write_padded: cutlass.Constexpr[bool],+ write_second: cutlass.Constexpr[bool],+ rotate: cutlass.Constexpr[bool],+ qhead_per_kvhead: cutlass.Constexpr[int],+ divisors: cutlass.Constexpr,+ query_chunks: cutlass.Constexpr[int],+ write_tail: cutlass.Constexpr[bool],+ rotate_key: cutlass.Constexpr[bool],+ shift_key: cutlass.Constexpr[bool],+ sink_tokens: cutlass.Constexpr[int],+):+ """Each row's query heads, then optionally the row's new K/V token.++ The first ``query_chunks`` CTAs of a row quantize its query; the rest+ write its K/V token into the BF16 tail. Neither side reads what the other+ writes, so they run side by side with no synchronization.+ """+ thread, _, _ = cute.arch.thread_idx()+ row, chunk, _ = cute.arch.block_idx()+ if const_expr(write_tail):+ if chunk >= query_chunks:+ position = seq_lens[row] - 1+ _write_tail_token(+ key, value, key_shift, tail_key, tail_value,+ row, row_to_slot[row], position, position - seqused_fp4[row],+ (chunk - query_chunks) * THREADS + thread,+ heads // qhead_per_kvhead, rest_k * 64, rotate_key, shift_key,+ sink_tokens,+ )+ else:+ _quantize_query_group(+ query, row_indices, query_fp4, query_scales,+ query_fp4_second, query_scales_second, query_padded,+ row, chunk * THREADS + thread,+ heads, rest_k, use_row_indices, write_padded, write_second,+ rotate, qhead_per_kvhead, divisors,+ )+ else:+ _quantize_query_group(+ query, row_indices, query_fp4, query_scales,+ query_fp4_second, query_scales_second, query_padded,+ row, chunk * THREADS + thread,+ heads, rest_k, use_row_indices, write_padded, write_second,+ rotate, qhead_per_kvhead, divisors,+ )+++def _quantize_query_group(+ query: cute.Tensor,+ row_indices: cute.Tensor,+ query_fp4: cute.Tensor,+ query_scales: cute.Tensor,+ query_fp4_second: cute.Tensor,+ query_scales_second: cute.Tensor,+ query_padded: cute.Tensor,+ row: Int32,+ item: Int32,+ heads: cutlass.Constexpr[int],+ rest_k: cutlass.Constexpr[int],+ use_row_indices: cutlass.Constexpr[bool],+ write_padded: cutlass.Constexpr[bool],+ write_second: cutlass.Constexpr[bool],+ rotate: cutlass.Constexpr[bool],+ qhead_per_kvhead: cutlass.Constexpr[int],+ divisors: cutlass.Constexpr,+):+ """One thread per 16-element scale group of one row's query heads.++ NVFP4's scale, codes and residual depend on one lane's sixteen values.+ MXFP4 shares the maximum and candidate costs with the adjacent lane;+ Hadamard and the atom's packed scale word also cross lanes. Threads past the+ last head load a clamped head and take part in those shuffles, so every+ lane of a warp reaches them; only their stores are skipped.+ """+ groups_per_head = const_expr(rest_k * GROUPS_PER_ATOM)+ active = item < heads * groups_per_head+ head = Int32(cutlass.select_(active, item // groups_per_head, heads - 1))+ group = item % groups_per_head++ source_row = row+ if const_expr(use_row_indices):+ source_row = row_indices[row]+ loaded = cute.make_rmem_tensor(+ cute.make_layout(SF_VEC_SIZE), cutlass.BFloat16+ )+ cute.autovec_copy(+ _group_slice(query[source_row, head, None], group, SF_VEC_SIZE),+ loaded,+ )+ values = cute.make_rmem_tensor(cute.make_layout(SF_VEC_SIZE), Float32)+ values.store(loaded.load().to(Float32))++ if const_expr(rotate):+ _hadamard(values, group, groups_per_head, groups_per_head * SF_VEC_SIZE)++ if const_expr(write_padded):+ padded = loaded+ if const_expr(rotate):+ padded = cute.make_rmem_tensor(+ cute.make_layout(SF_VEC_SIZE), cutlass.BFloat16+ )+ padded.store(values.load().to(cutlass.BFloat16))+ if active:+ cute.autovec_copy(+ padded,+ _group_slice(query_padded[row, 0, head, None], group, SF_VEC_SIZE),+ )++ if const_expr(isinstance(divisors, str)):+ scale, low, high = _select_mx_query_group(values, divisors)+ else:+ scale, low, high = select_group_scale(values, 0, SF_VEC_SIZE, divisors)+ _store_term(+ scale, low, high, query_fp4, query_scales,+ row, head, group, active, qhead_per_kvhead, isinstance(divisors, str),+ )+ if const_expr(write_second):+ residual = cute.make_rmem_tensor(+ cute.make_layout(SF_VEC_SIZE), Float32+ )+ group_residual(values, residual, 0, scale, low, high, SF_VEC_SIZE)+ if const_expr(isinstance(divisors, str)):+ scale, low, high = _select_mx_query_group(residual, divisors)+ else:+ scale, low, high = select_group_scale(+ residual, 0, SF_VEC_SIZE, divisors+ )+ _store_term(+ scale, low, high, query_fp4_second, query_scales_second,+ row, head, group, active, qhead_per_kvhead, isinstance(divisors, str),+ )+++def _launch_quantize_query(+ query_ptr: cute.Pointer,+ row_indices_ptr: cute.Pointer,+ query_fp4_ptr: cute.Pointer,+ query_scales_ptr: cute.Pointer,+ query_fp4_second_ptr: cute.Pointer,+ query_scales_second_ptr: cute.Pointer,+ query_padded_ptr: cute.Pointer,+ query_shape: Tuple[Int32, Int32, Int32],+ query_strides: Tuple[Int32, Int32, Int32],+ row_indices_shape: Tuple[Int32],+ fp4_shape: Tuple[Int32, Int32, Int32, Int32],+ scales_shape: Tuple[+ Int32, Int32, Int32, Int32, Int32, Int32, Int32+ ],+ scales_strides: Tuple[+ Int32, Int32, Int32, Int32, Int32, Int32, Int32+ ],+ padded_shape: Tuple[Int32, Int32, Int32, Int32],+ key_ptr: cute.Pointer,+ value_ptr: cute.Pointer,+ key_shift_ptr: cute.Pointer,+ tail_key_ptr: cute.Pointer,+ tail_value_ptr: cute.Pointer,+ row_to_slot_ptr: cute.Pointer,+ seq_lens_ptr: cute.Pointer,+ seqused_fp4_ptr: cute.Pointer,+ key_shape: Tuple[Int32, Int32, Int32],+ key_strides: Tuple[Int32, Int32, Int32],+ value_strides: Tuple[Int32, Int32, Int32],+ tail_shape: Tuple[Int32, Int32, Int32, Int32],+ tail_strides: Tuple[Int32, Int32, Int32, Int32],+ rows: Int32,+ heads: cutlass.Constexpr[int],+ rest_k: cutlass.Constexpr[int],+ use_row_indices: cutlass.Constexpr[bool],+ write_padded: cutlass.Constexpr[bool],+ write_second: cutlass.Constexpr[bool],+ rotate: cutlass.Constexpr[bool],+ qhead_per_kvhead: cutlass.Constexpr[int],+ divisors: cutlass.Constexpr,+ write_tail: cutlass.Constexpr[bool],+ rotate_key: cutlass.Constexpr[bool],+ shift_key: cutlass.Constexpr[bool],+ sink_tokens: cutlass.Constexpr[int],+ stream: cuda.CUstream,+):+ query = cute.make_tensor(+ query_ptr,+ cute.make_layout(query_shape, stride=query_strides),+ )+ row_indices = cute.make_tensor(+ row_indices_ptr,+ cute.make_ordered_layout(row_indices_shape, order=(0,)),+ )+ query_fp4 = cute.make_tensor(+ query_fp4_ptr,+ cute.make_ordered_layout(+ fp4_shape,+ order=tuple(range(len(fp4_shape) - 1, -1, -1)),+ ),+ )+ query_scales = cute.make_tensor(+ query_scales_ptr,+ cute.make_layout(scales_shape, stride=scales_strides),+ )+ query_fp4_second = cute.make_tensor(+ query_fp4_second_ptr,+ cute.make_ordered_layout(+ fp4_shape,+ order=tuple(range(len(fp4_shape) - 1, -1, -1)),+ ),+ )+ query_scales_second = cute.make_tensor(+ query_scales_second_ptr,+ cute.make_layout(scales_shape, stride=scales_strides),+ )+ query_padded = cute.make_tensor(+ query_padded_ptr,+ cute.make_ordered_layout(+ padded_shape,+ order=tuple(range(len(padded_shape) - 1, -1, -1)),+ ),+ )++ key = cute.make_tensor(+ key_ptr, cute.make_layout(key_shape, stride=key_strides)+ )+ value = cute.make_tensor(+ value_ptr, cute.make_layout(key_shape, stride=value_strides)+ )+ key_shift = cute.make_tensor(+ key_shift_ptr,+ cute.make_ordered_layout(+ (key_shape[1], key_shape[2]), order=(1, 0)+ ),+ )+ tail_key = cute.make_tensor(+ tail_key_ptr, cute.make_layout(tail_shape, stride=tail_strides)+ )+ tail_value = cute.make_tensor(+ tail_value_ptr, cute.make_layout(tail_shape, stride=tail_strides)+ )+ row_to_slot = cute.make_tensor(+ row_to_slot_ptr, cute.make_ordered_layout((rows,), order=(0,))+ )+ seq_lens = cute.make_tensor(+ seq_lens_ptr, cute.make_ordered_layout((rows,), order=(0,))+ )+ seqused_fp4 = cute.make_tensor(+ seqused_fp4_ptr, cute.make_ordered_layout((rows,), order=(0,))+ )++ query_chunks = const_expr(+ (heads * rest_k * GROUPS_PER_ATOM + THREADS - 1) // THREADS+ )+ # K's groups padded to whole warps, then V's.+ tail_items = const_expr(heads // qhead_per_kvhead * rest_k * GROUPS_PER_ATOM)+ tail_chunks = const_expr(+ ((tail_items + 31) // 32 * 32 + tail_items + THREADS - 1) // THREADS+ if write_tail+ else 0+ )+ _quantize_query_kernel(+ query,+ row_indices,+ query_fp4,+ query_scales,+ query_fp4_second,+ query_scales_second,+ query_padded,+ key,+ value,+ key_shift,+ tail_key,+ tail_value,+ row_to_slot,+ seq_lens,+ seqused_fp4,+ const_expr(heads),+ const_expr(rest_k),+ const_expr(use_row_indices),+ const_expr(write_padded),+ const_expr(write_second),+ const_expr(rotate),+ const_expr(qhead_per_kvhead),+ const_expr(divisors),+ const_expr(query_chunks),+ const_expr(write_tail),+ const_expr(rotate_key),+ const_expr(shift_key),+ const_expr(sink_tokens),+ ).launch(+ grid=(rows, query_chunks + tail_chunks, 1),+ block=(THREADS, 1, 1),+ stream=stream,+ )+++_launch_quantize_query.compile_cache = {}+++def quantize_decode_q_to_padded_fp4(+ query: torch.Tensor,+ query_fp4_out: torch.Tensor,+ query_scales_out: torch.Tensor,+ query_padded_out: Optional[torch.Tensor] = None,+ *,+ query_fp4_out_2: Optional[torch.Tensor] = None,+ query_scales_out_2: Optional[torch.Tensor] = None,+ row_indices: Optional[torch.Tensor] = None,+ heads_kv: Optional[int] = None,+ scale_mode: str = DEFAULT_SCALE_MODE,+ rotation: str = "none",+ tail_write: Optional[DecodeTailWrite] = None,+) -> None:+ """Quantize selected BF16 query rows into preallocated outputs.++ ``scale_mode`` names the block-scale candidates; see :mod:`.scale_mode`.+ ``rotation="hadamard"`` first multiplies each head by the normalized+ Sylvester Hadamard in FP32; both terms are quantized from those FP32+ values, and the padded copy holds them rounded to BF16. Both arguments+ join the compile cache key, so switching costs one compilation and nothing+ per call. ``tail_write`` also places each row's new K/V token in the BF16+ tail from extra CTAs of the same launch; see :class:`DecodeTailWrite`.+ """+ divisors = scale_rule(scale_mode)+ scale_group_size = sf_vec_size(scale_mode)+ if rotation not in QUERY_ROTATIONS:+ raise ValueError(+ f"rotation must be one of {QUERY_ROTATIONS}, got {rotation!r}"+ )+ if query.dtype is not torch.bfloat16 or not query.is_cuda:+ raise ValueError("query must be a BF16 CUDA tensor")+ if query.ndim != 3 or query.stride(-1) != 1:+ raise ValueError("query must have shape [rows, heads, head_dim]")+ if (+ query.data_ptr() % 16+ or query.stride(0) % 8+ or query.stride(1) % 8+ ):+ raise ValueError(+ "every query head vector must start on a 16-byte boundary: "+ "the data pointer must be 16-byte aligned and the row and head "+ "strides multiples of 8 elements"+ )++ source_rows, heads, head_dim = query.shape+ heads_kv = heads if heads_kv is None else heads_kv+ if heads_kv < 1 or heads % heads_kv:+ raise ValueError("query heads must be divisible by heads_kv")+ qhead_per_kvhead = heads // heads_kv+ rows = query_fp4_out.shape[0]+ if head_dim % (GROUPS_PER_ATOM * scale_group_size) != 0:+ raise ValueError(+ f"query head_dim must be divisible by {GROUPS_PER_ATOM * scale_group_size}"+ )+ rotate = rotation == "hadamard"+ if rotate and (+ head_dim & (head_dim - 1) or head_dim // SF_VEC_SIZE > 32+ ):+ raise ValueError(+ "the fused Hadamard needs a power-of-two head_dim of at most "+ f"{32 * SF_VEC_SIZE}, got {head_dim}"+ )+ if row_indices is None:+ if source_rows != rows:+ raise ValueError("query rows must match output rows")+ elif (+ row_indices.dtype is not torch.int32+ or not row_indices.is_cuda+ or row_indices.device != query.device+ or not row_indices.is_contiguous()+ or row_indices.shape != (rows,)+ ):+ raise ValueError(+ "row_indices must be contiguous INT32 CUDA with shape [rows]"+ )++ expected_fp4_shape = (+ rows,+ 1,+ heads,+ head_dim // 2,+ )+ expected_scales_shape = (+ 32,+ 4,+ 1,+ 4,+ head_dim // (GROUPS_PER_ATOM * scale_group_size),+ heads_kv,+ rows,+ )+ write_second = (+ query_fp4_out_2 is not None or query_scales_out_2 is not None+ )+ if (query_fp4_out_2 is None) != (query_scales_out_2 is None):+ raise ValueError(+ "query_fp4_out_2 and query_scales_out_2 must be provided together"+ )+ if (+ tuple(query_fp4_out.shape) != expected_fp4_shape+ or query_fp4_out.dtype is not torch.float4_e2m1fn_x2+ or not query_fp4_out.is_cuda+ or query_fp4_out.device != query.device+ or not query_fp4_out.is_contiguous()+ ):+ raise ValueError(+ f"query_fp4_out must have shape {expected_fp4_shape}"+ )+ if (+ tuple(query_scales_out.shape) != expected_scales_shape+ or query_scales_out.dtype is not torch.uint8+ or not query_scales_out.is_cuda+ or query_scales_out.device != query.device+ ):+ raise ValueError(+ f"query_scales_out must have shape {expected_scales_shape}"+ )+ if write_second:+ assert query_fp4_out_2 is not None+ assert query_scales_out_2 is not None+ if (+ tuple(query_fp4_out_2.shape) != expected_fp4_shape+ or query_fp4_out_2.dtype is not torch.float4_e2m1fn_x2+ or not query_fp4_out_2.is_cuda+ or query_fp4_out_2.device != query.device+ or not query_fp4_out_2.is_contiguous()+ ):+ raise ValueError(+ f"query_fp4_out_2 must have shape {expected_fp4_shape}"+ )+ if (+ tuple(query_scales_out_2.shape) != expected_scales_shape+ or query_scales_out_2.dtype is not torch.uint8+ or not query_scales_out_2.is_cuda+ or query_scales_out_2.device != query.device+ ):+ raise ValueError(+ f"query_scales_out_2 must have shape {expected_scales_shape}"+ )++ write_padded = query_padded_out is not None+ if write_padded:+ if (+ query_padded_out.ndim != 4+ or query_padded_out.shape[0] < rows+ or tuple(query_padded_out.shape[1:])+ != (PAGE_SIZE, heads, head_dim)+ or query_padded_out.dtype is not torch.bfloat16+ or not query_padded_out.is_cuda+ or query_padded_out.device != query.device+ or not query_padded_out.is_contiguous()+ ):+ raise ValueError(+ "query_padded_out must have shape "+ f"[at least {rows}, {PAGE_SIZE}, {heads}, {head_dim}]"+ )++ stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream)+ query_ptr = make_ptr(+ cutlass.BFloat16,+ query.data_ptr(),+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ if row_indices is None:+ row_indices_ptr = make_ptr(+ cutlass.Int32,+ query.data_ptr(),+ cute.AddressSpace.gmem,+ assumed_align=4,+ )+ else:+ row_indices_ptr = make_ptr(+ cutlass.Int32,+ row_indices.data_ptr(),+ cute.AddressSpace.gmem,+ assumed_align=4,+ )+ fp4_ptr = make_ptr(+ cutlass.Uint8,+ query_fp4_out.data_ptr(),+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ scales_ptr = make_ptr(+ cutlass.Uint8,+ query_scales_out.data_ptr(),+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ fp4_second_ptr = make_ptr(+ cutlass.Uint8,+ (+ query_fp4_out_2.data_ptr()+ if query_fp4_out_2 is not None+ else query_fp4_out.data_ptr()+ ),+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ scales_second_ptr = make_ptr(+ cutlass.Uint8,+ (+ query_scales_out_2.data_ptr()+ if query_scales_out_2 is not None+ else query_scales_out.data_ptr()+ ),+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ if query_padded_out is None:+ padded_ptr = query_ptr+ padded_shape = (+ Int32(rows),+ Int32(1),+ Int32(heads),+ Int32(head_dim),+ )+ else:+ padded_ptr = make_ptr(+ cutlass.BFloat16,+ query_padded_out.data_ptr(),+ cute.AddressSpace.gmem,+ assumed_align=16,+ )+ padded_shape = tuple(+ Int32(value) for value in query_padded_out.shape+ )++ write_tail = tail_write is not None+ if write_tail:+ tail_args = _tail_write_arguments(+ tail_write, rows, heads_kv, head_dim, query.device, row_indices+ )+ rotate_key = bool(tail_write.rotate_key)+ shift_key = tail_write.key_shift is not None+ sink_tokens = int(tail_write.sink_tokens)+ else:+ # Never dereferenced: every use sits under the write_tail constexpr.+ tail_args = (+ query_ptr, query_ptr,+ make_ptr(cutlass.Float32, query.data_ptr(), cute.AddressSpace.gmem,+ assumed_align=16),+ query_ptr, query_ptr,+ row_indices_ptr, row_indices_ptr, row_indices_ptr,+ (Int32(rows), Int32(heads_kv), Int32(head_dim)),+ (Int32(0), Int32(0), Int32(1)),+ (Int32(0), Int32(0), Int32(1)),+ (Int32(1), Int32(1), Int32(heads_kv), Int32(head_dim)),+ (Int32(0), Int32(0), Int32(0), Int32(1)),+ )+ rotate_key = shift_key = False+ sink_tokens = 0++ query_shape = tuple(Int32(value) for value in query.shape)+ query_strides = tuple(Int32(value) for value in query.stride())+ row_indices_shape = (Int32(rows),)+ fp4_shape = tuple(Int32(value) for value in expected_fp4_shape)+ scales_shape = tuple(Int32(value) for value in expected_scales_shape)+ scales_strides = tuple(+ Int32(value) for value in query_scales_out.stride()+ )+ # This counts four-lane groups, independently of the stored scale atoms.+ # MX still gives each lane sixteen values; its scale tile has half as+ # many atoms, but its Hadamard and tail-write geometry stay unchanged.+ rest_k = head_dim // 64+ use_row_indices = row_indices is not None++ compile_args = (+ query_ptr,+ row_indices_ptr,+ fp4_ptr,+ scales_ptr,+ fp4_second_ptr,+ scales_second_ptr,+ padded_ptr,+ query_shape,+ query_strides,+ row_indices_shape,+ fp4_shape,+ scales_shape,+ scales_strides,+ padded_shape,+ *tail_args,+ Int32(rows),+ heads,+ rest_k,+ use_row_indices,+ write_padded,+ write_second,+ rotate,+ qhead_per_kvhead,+ divisors,+ write_tail,+ rotate_key,+ shift_key,+ sink_tokens,+ stream,+ )+ compile_key = (+ torch.cuda.current_device(),+ heads,+ rest_k,+ use_row_indices,+ write_padded,+ write_second,+ rotate,+ qhead_per_kvhead,+ divisors,+ write_tail,+ rotate_key,+ shift_key,+ sink_tokens,+ )+ compiled = _launch_quantize_query.compile_cache.get(compile_key)+ if compiled is None:+ compiled = cute.compile(_launch_quantize_query, *compile_args)+ _launch_quantize_query.compile_cache[compile_key] = compiled++ compiled(+ query_ptr,+ row_indices_ptr,+ fp4_ptr,+ scales_ptr,+ fp4_second_ptr,+ scales_second_ptr,+ padded_ptr,+ query_shape,+ query_strides,+ row_indices_shape,+ fp4_shape,+ scales_shape,+ scales_strides,+ padded_shape,+ *tail_args,+ Int32(rows),+ stream,+ )+++def _tail_write_arguments(+ tail_write: DecodeTailWrite,+ rows: int,+ heads_kv: int,+ head_dim: int,+ device: torch.device,+ row_indices: Optional[torch.Tensor],+) -> tuple:+ """Check a :class:`DecodeTailWrite` and turn it into launch arguments."""+ if row_indices is not None:+ raise ValueError("tail_write needs the query rows in batch order")+ key, value = tail_write.key, tail_write.value+ for name, tensor in (("key", key), ("value", value)):+ if (+ tensor.dtype is not torch.bfloat16+ or tensor.device != device+ or tensor.ndim != 3+ or tensor.shape[0] < rows+ or tuple(tensor.shape[1:]) != (heads_kv, head_dim)+ or tensor.stride(2) != 1+ or tensor.data_ptr() % 16+ or tensor.stride(0) % 8+ or tensor.stride(1) % 8+ ):+ raise ValueError(+ f"tail_write.{name} must be BF16 [>= {rows}, {heads_kv}, "+ f"{head_dim}] with 16-byte aligned head vectors"+ )+ tail_key, tail_value = tail_write.tail_key, tail_write.tail_value+ tail_shape = (PAGE_SIZE + tail_write.sink_tokens, heads_kv, head_dim)+ for name, tensor in (("tail_key", tail_key), ("tail_value", tail_value)):+ if (+ tensor.dtype is not torch.bfloat16+ or tensor.device != device+ or tensor.ndim != 4+ or tuple(tensor.shape[1:]) != tail_shape+ or tensor.stride() != tail_key.stride()+ or tensor.stride(3) != 1+ or tensor.data_ptr() % 16+ or any(stride % 8 for stride in tensor.stride()[:3])+ ):+ raise ValueError(+ f"tail_write.{name} must be BF16 [slots, *{tail_shape}] with "+ "the tail key's strides and 16-byte aligned head vectors"+ )+ for name in ("row_to_slot", "seq_lens", "seqused_fp4"):+ tensor = getattr(tail_write, name)+ if (+ tensor.dtype is not torch.int32+ or tensor.device != device+ or not tensor.is_contiguous()+ or tensor.ndim != 1+ or tensor.shape[0] < rows+ ):+ raise ValueError(+ f"tail_write.{name} must be contiguous INT32 with >= {rows} rows"+ )+ shift = tail_write.key_shift+ if shift is not None and (+ shift.dtype is not torch.float32+ or shift.device != device+ or tuple(shift.shape) != (heads_kv, head_dim)+ or not shift.is_contiguous()+ ):+ raise ValueError(+ f"tail_write.key_shift must be contiguous FP32 [{heads_kv}, {head_dim}]"+ )+ if tail_write.rotate_key and head_dim & (head_dim - 1):+ raise ValueError("the K rotation needs a power-of-two head_dim")++ def pointer(dtype, tensor, align):+ return make_ptr(dtype, tensor.data_ptr(), cute.AddressSpace.gmem,+ assumed_align=align)++ return (+ pointer(cutlass.BFloat16, key, 16),+ pointer(cutlass.BFloat16, value, 16),+ pointer(cutlass.Float32, key if shift is None else shift, 16),+ pointer(cutlass.BFloat16, tail_key, 16),+ pointer(cutlass.BFloat16, tail_value, 16),+ pointer(cutlass.Int32, tail_write.row_to_slot, 4),+ pointer(cutlass.Int32, tail_write.seq_lens, 4),+ pointer(cutlass.Int32, tail_write.seqused_fp4, 4),+ tuple(Int32(value) for value in key.shape),+ tuple(Int32(value) for value in key.stride()),+ tuple(Int32(value) for value in value.stride()),+ tuple(Int32(value) for value in tail_key.shape),+ tuple(Int32(value) for value in tail_key.stride()),+ )diff --git a/src/nvfp4_decode_kernel/reference.py b/src/nvfp4_decode_kernel/reference.pynew file mode 100644index 0000000..24e78b9--- /dev/null+++ b/src/nvfp4_decode_kernel/reference.py@@ -0,0 +1,871 @@+"""PyTorch reference operations for the kernel's NVFP4 quantization.++This mirrors the CuTeDSL quantizers arithmetically, not merely closely. E2M1+rounding breaks ties towards the even code as ``cvt.rn.satfinite.e2m1x2.f32``+does, a scale is the reciprocal multiply the kernel performs rather than a+division, and the squared error a candidate is judged by accumulates through+fused multiply-add in element order. Each of those is a last-place difference+on its own, but they feed an argmin whose losers are sometimes a hair behind,+so any one of them diverging turns into the two implementations storing+different bytes.+"""++from __future__ import annotations++import math++import torch++from .p_scale import P_SMALL_GROUP_AMAX+from .scale_mode import DEFAULT_SCALE_MODE, candidates+++# Magnitudes an E2M1 code can take, indexed by the code's low three bits.+_E2M1_MAGNITUDES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)++# Midpoint ``m`` separates codes ``m`` and ``m + 1``, so round-to-nearest-even+# sends a tie down at an even midpoint and up at an odd one.+_E2M1_MIDPOINTS = (0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0)++# E4M3's largest finite value. A scale is clamped here before being cast,+# because ``cvt.rn.satfinite.e4m3x2.f32`` saturates and Torch does not agree+# with it across versions — 2.11 returns NaN for anything that rounds past the+# top, 2.13 saturates. Clamping first reproduces the instruction on either: a+# non-negative value that would round past 448 is exactly one that saturates+# there, and a scale is never negative.+_E4M3_MAX = 448.0++_PROBABILITY_SCALE_MODES = {+ "two_level": (6.0,),+ "amax2_or_amax6_c8": (2.0, 6.0),+ "amax2": (2.0,),+ "amax3": (3.0,),+ "amax4": (4.0,),+ # Map a group's maximum to each useful interior/top E2M1 rung. Keep the+ # production amax/6 choice first so it wins exact ties.+ "e2m1_max_grid": (6.0, 4.0, 3.0, 2.0),+}+++def _e2m1_boundaries(device: torch.device) -> torch.Tensor:+ """Midpoints arranged so that ``bucketize`` breaks ties towards even.++ ``bucketize`` sends a value equal to a boundary into the lower bucket, so+ the odd midpoints are moved one float below the tie to send it the other+ way. One float, in float32, because that is the width the comparison+ happens in — anything wider rounds back onto the tie.+ """+ midpoints = torch.tensor(+ _E2M1_MIDPOINTS, dtype=torch.float32, device=device+ )+ below = torch.nextafter(midpoints, torch.zeros_like(midpoints))+ odd = torch.arange(midpoints.numel(), device=device) % 2 == 1+ return torch.where(odd, below, midpoints)+++def round_e2m1(x: torch.Tensor) -> torch.Tensor:+ """Round values to the nearest representable E2M1 value, ties to even.++ Magnitudes past 6 saturate there rather than overflowing, which is the+ ``satfinite`` half of the instruction's name and what the zero-scale guard+ in :func:`nvfp4_quantize` relies on.+ """+ magnitudes = torch.tensor(+ _E2M1_MAGNITUDES, dtype=torch.float32, device=x.device+ )+ indices = torch.bucketize(+ x.abs(), _e2m1_boundaries(x.device), right=False, out_int32=True+ )+ return torch.sign(x) * magnitudes[indices]+++def e2m1_codes(x: torch.Tensor) -> torch.Tensor:+ """The four-bit code each value rounds to, sign in bit 3.++ Same bucketing as :func:`round_e2m1`, which is the point: a writer that+ derived the code independently could disagree with the reconstruction in+ the one place ties are broken. Negative values that round to zero get code+ 8, the negative zero the hardware conversion also emits; it reads back as+ the same magnitude, so a byte comparison against a path that emitted 0+ there is comparing an encoding rather than a value.+ """+ indices = torch.bucketize(+ x.abs(), _e2m1_boundaries(x.device), right=False, out_int32=True+ )+ return (indices | torch.where(x < 0, 8, 0)).to(torch.uint8)+++def _accumulate_squares(difference: torch.Tensor) -> torch.Tensor:+ """A group's squared error, accumulated the way the kernel accumulates it.++ The kernel spells each step ``fma.rn.f32``: one rounding per element,+ taken in element order. Both halves of that matter. A reduction free to+ reassociate lands a last place away, and so does squaring into float32+ before adding, and either is enough to swap two candidates that are+ otherwise level.++ Float64 reproduces the instruction rather than approximating it. The+ product of two float32 values is exact in float64, and rounding the+ float64 sum back to float32 agrees with a single rounding of the exact+ value, because float64's 53 bits clear the 2p+2 that makes the double+ rounding harmless at p=24.+ """+ # An MX group is owned by two lanes, each accumulating sixteen values.+ # Keep that single final addition in the reference as well as in K/V's+ # one-thread quantizer, so all three select the same near-tie exponent.+ if difference.shape[-1] == 32:+ return _accumulate_squares(difference[..., :16]) + _accumulate_squares(+ difference[..., 16:]+ )+ total = torch.zeros_like(difference[..., :1])+ wide = difference.double()+ for element in range(difference.shape[-1]):+ term = wide[..., element : element + 1]+ total = (total.double() + term * term).float()+ return total+++def _apply_candidate(+ groups: torch.Tensor,+ amax: torch.Tensor,+ divisor: float,+) -> tuple[torch.Tensor, torch.Tensor]:+ """One candidate's scale and its round trip."""+ # The kernel holds 1/divisor as a float32 constant and multiplies by it, so+ # dividing here would differ in the last place on some amax and, through+ # E4M3's three mantissa bits, occasionally in the byte that gets stored.+ reciprocal = torch.tensor(1.0 / divisor, dtype=torch.float32).item()+ scale = (+ (amax * reciprocal).clamp(max=_E4M3_MAX).to(torch.float8_e4m3fn).float()+ )+ # The kernel's guard against a scale that flushed to zero. Every nonzero+ # element is then thrown past the top of the grid, where E2M1 saturates at+ # 6, and multiplying by the zero scale brings the group back to zero. Zero+ # is what the decode path reads back in that case, so it is what a+ # reference has to produce.+ return scale, round_e2m1(groups * (1.0 / (scale + 1.0e-30))) * scale+++def nvfp4_quantize(+ x: torch.Tensor,+ group: int = 16,+ scale_mode: str = DEFAULT_SCALE_MODE,+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:+ """Quantize groups on the last axis, each with the scale that costs least.++ Returns the round-tripped values, the E4M3 scale every group settled on,+ and which candidate that was as an index into ``scale_mode``'s tuple.++ No candidate is screened for range. E4M3 saturates at 448 and flushes below+ ``2**-9``, so a divisor too small for the group leaves a scale that cannot+ represent it and an error that says as much. Judging every candidate by the+ scale it would actually store lets the argmin drop those without a branch,+ and it never drops one that would have won.+ """+ if group <= 0:+ raise ValueError(f"group must be positive, got {group}")+ if x.shape[-1] % group != 0:+ raise ValueError(+ f"last dimension {x.shape[-1]} must be divisible by group {group}"+ )++ groups = x.float().reshape(*x.shape[:-1], x.shape[-1] // group, group)+ amax = groups.abs().amax(dim=-1, keepdim=True)++ best_scale, best_values, best_error, winner = None, None, None, None+ divisors = candidates(scale_mode)+ for index, divisor in enumerate(divisors):+ scale, values = _apply_candidate(groups, amax, divisor)+ if len(divisors) == 1:+ return (+ values.reshape_as(x),+ scale.squeeze(-1).to(torch.float8_e4m3fn),+ torch.zeros_like(amax.squeeze(-1), dtype=torch.int64),+ )+ # An E2M1 magnitude carries two significant bits and an E4M3 scale+ # four, so their product is exact in float32 and this subtraction is+ # the kernel's ``fma(magnitude, scale, -x)`` to the bit.+ error = _accumulate_squares(values - groups)+ if best_error is None:+ best_scale, best_values, best_error = scale, values, error+ winner = torch.zeros_like(error, dtype=torch.int64)+ continue+ # Strictly better, so a tie leaves the earlier candidate in place.+ better = error < best_error+ best_scale = torch.where(better, scale, best_scale)+ best_values = torch.where(better, values, best_values)+ best_error = torch.where(better, error, best_error)+ winner = torch.where(better, torch.full_like(winner, index), winner)++ return (+ best_values.reshape_as(x),+ best_scale.squeeze(-1).to(torch.float8_e4m3fn),+ winner.squeeze(-1),+ )+++def nvfp4_round_trip(+ x: torch.Tensor,+ group: int = 16,+ scale_mode: str = DEFAULT_SCALE_MODE,+) -> torch.Tensor:+ """Quantize and dequantize groups on the last axis with NVFP4."""+ values, _, _ = nvfp4_quantize(x, group=group, scale_mode=scale_mode)+ return values+++def hadamard_rotate(x: torch.Tensor) -> torch.Tensor:+ """The last axis times the normalized Sylvester Hadamard, as the kernels do.++ FP32 butterflies ``(a, c) -> (a + c, a - c)`` over the index bits, lowest+ first, then one multiply by ``1 / sqrt(n)``. That is the query quantizer's+ and the K writer's order and rounding, so their FP32 results match this+ bit for bit; a matrix product sums in another order and does not.+ """+ n = x.shape[-1]+ if n <= 0 or n & (n - 1):+ raise ValueError(f"Hadamard order must be a power of two, got {n}")+ leading = x.shape[:-1]+ values = x.float()+ stride = 1+ while stride < n:+ pairs = values.reshape(*leading, n // (2 * stride), 2, stride)+ low, high = pairs[..., 0, :], pairs[..., 1, :]+ values = torch.stack((low + high, low - high), dim=-2)+ stride *= 2+ inverse = torch.tensor(+ 1.0 / math.sqrt(n), dtype=torch.float32, device=values.device+ )+ return values.reshape(*leading, n) * inverse+++def nvfp4_two_term(+ x: torch.Tensor,+ group: int = 16,+ scale_mode: str = DEFAULT_SCALE_MODE,+) -> tuple[torch.Tensor, torch.Tensor]:+ """Return the two NVFP4 reconstructions whose sum approximates ``x``."""+ first = nvfp4_round_trip(x, group=group, scale_mode=scale_mode)+ second = nvfp4_round_trip(+ x.float() - first,+ group=group,+ scale_mode=scale_mode,+ )+ return first, second+++# OCP MXFP4: 32 values share one E8M0 scale, a power of two 2**e stored as the+# byte e + 127 (255 is NaN). E2M1's largest value is 1.5 * 2**2, so the OCP+# rule e = floor(log2(amax)) - 2 maps amax into [4, 8) and saturates what lands+# above 6.+MX_GROUP = 32+MX_SCALE_RULES = ("ocp", "round_up", "search")+_E2M1_MAX_EXPONENT = 2+_E8M0_MIN_EXPONENT = -127+_E8M0_MAX_EXPONENT = 127+++def _mx_round_trip(groups: torch.Tensor, exponent: torch.Tensor) -> torch.Tensor:+ """``groups`` rounded on the power-of-two scale ``2**exponent``."""+ scale = torch.ldexp(torch.ones_like(exponent, dtype=torch.float32), exponent)+ return round_e2m1(groups / scale) * scale+++def mxfp4_quantize(+ x: torch.Tensor, rule: str = "ocp"+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:+ """Quantize groups of 32 on the last axis to MXFP4.++ ``rule`` picks each group's exponent: ``ocp`` is the OCP specification's+ ``floor(log2(amax)) - 2``, ``round_up`` the smallest exponent that keeps+ the group maximum at or below 6, and ``search`` whichever of ``ocp`` and+ ``ocp + 1`` reconstructs the group with less squared error (ties keep+ ``ocp``). ``round_up`` is always one of those two, so ``search`` is never+ worse than either fixed rule. An all-zero group gets the smallest scale,+ because E8M0 has no zero.++ Returns the round-tripped values, the E8M0 bytes, and which exponent won+ (0 for ``ocp``, 1 for ``ocp + 1``).+ """+ if rule not in MX_SCALE_RULES:+ raise ValueError(f"rule must be one of {MX_SCALE_RULES}, got {rule!r}")+ if x.shape[-1] % MX_GROUP:+ raise ValueError(+ f"last dimension {x.shape[-1]} must be divisible by {MX_GROUP}"+ )+ groups = x.float().reshape(*x.shape[:-1], x.shape[-1] // MX_GROUP, MX_GROUP)+ amax = groups.abs().amax(dim=-1, keepdim=True)+ # frexp gives amax = m * 2**k with m in [0.5, 1), so floor(log2(amax)) is+ # k - 1 exactly, with no logarithm to round.+ _, exponent = torch.frexp(amax)+ ocp = (exponent.to(torch.int32) - 1 - _E2M1_MAX_EXPONENT).clamp(+ _E8M0_MIN_EXPONENT, _E8M0_MAX_EXPONENT+ )+ ocp = torch.where(amax > 0, ocp, torch.full_like(ocp, _E8M0_MIN_EXPONENT))+ ocp_values = _mx_round_trip(groups, ocp)+ if rule == "ocp":+ winner = torch.zeros_like(ocp, dtype=torch.int64)+ else:+ doubled = (ocp + 1).clamp(max=_E8M0_MAX_EXPONENT)+ doubled_values = _mx_round_trip(groups, doubled)+ if rule == "round_up":+ ocp_scale = torch.ldexp(torch.ones_like(amax), ocp)+ choose = amax > 6.0 * ocp_scale+ else:+ choose = _accumulate_squares(doubled_values - groups) < _accumulate_squares(+ ocp_values - groups+ )+ winner = choose.to(torch.int64)+ ocp = torch.where(choose, doubled, ocp)+ ocp_values = torch.where(choose, doubled_values, ocp_values)+ codes = (ocp + 127).to(torch.uint8)+ return ocp_values.reshape_as(x), codes.squeeze(-1), winner.squeeze(-1)+++def mxfp4_round_trip(x: torch.Tensor, rule: str = "ocp") -> torch.Tensor:+ """Quantize and dequantize groups of 32 on the last axis with MXFP4."""+ values, _, _ = mxfp4_quantize(x, rule)+ return values+++def mxfp4_probability_quantize(probability: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:+ """P-specific adjacent-exponent search, including a group-mass penalty.++ Reproduce the warp's butterfly reduction and final FMA when comparing+ candidates. Return reconstructed values and one E8M0 byte per group.+ """+ baseline, codes, _ = mxfp4_quantize(probability, "round_up")+ groups = probability.float().reshape(*probability.shape[:-1], -1, 32)+ exponent = codes.int().unsqueeze(-1) - 127+ smaller = (exponent - 1).clamp(min=-127)+ rival = _mx_round_trip(groups, smaller)++ def cost(values):+ difference = values - groups+ squared, mass = difference.square(), difference+ for _ in range(5):+ squared = squared[..., 0::2] + squared[..., 1::2]+ mass = mass[..., 0::2] + mass[..., 1::2]+ return ((mass / 32).double() * mass.double() + squared.double()).float()++ baseline = baseline.reshape_as(groups)+ choose = cost(rival) < cost(baseline)+ values = torch.where(choose, rival, baseline).reshape_as(probability)+ selected = torch.where(choose, smaller, exponent).squeeze(-1)+ return values, (selected + 127).to(torch.uint8)+++FP4_FORMATS = ("nvfp4", "mxfp4")+++def fp4_round_trip(x: torch.Tensor, fmt: str, scale_mode: str) -> torch.Tensor:+ """Round trip on the last axis: NVFP4 groups of 16 with ``scale_mode`` as an+ NVFP4 candidate set, or MXFP4 groups of 32 with ``scale_mode`` as an MX rule."""+ if fmt == "nvfp4":+ return nvfp4_round_trip(x, scale_mode=scale_mode)+ if fmt == "mxfp4":+ return mxfp4_round_trip(x, scale_mode)+ raise ValueError(f"fmt must be one of {FP4_FORMATS}, got {fmt!r}")+++def fp4_probability_round_trip(+ probability: torch.Tensor, fmt: str, scale_mode: str = "amax6"+) -> torch.Tensor:+ """The decode kernel's P operand: NVFP4 per :func:`nvfp4_probability_quantize`,+ or MXFP4 with adjacent-exponent element/mass-error search."""+ if fmt == "nvfp4":+ values, _ = nvfp4_probability_quantize(+ probability, group=16, scale_mode=scale_mode+ )+ return values+ if fmt == "mxfp4":+ return mxfp4_probability_quantize(probability)[0]+ raise ValueError(f"fmt must be one of {FP4_FORMATS}, got {fmt!r}")+++def nvfp4_probability_quantize(+ x: torch.Tensor,+ *,+ group: int = 16,+ scale_mode: str = "amax6",+) -> tuple[torch.Tensor, torch.Tensor]:+ """Mirror the decode kernel's P-specific scale/code ordering.++ The online softmax normalizes P with the unrounded FP32 ``amax/divisor``+ before converting the normalized value to E2M1. Only the scale consumed+ by the P×V MMA is rounded to E4M3. This differs slightly from cached K/V,+ whose code is selected using the already-rounded stored scale.+ """+ if group <= 0 or x.shape[-1] % group:+ raise ValueError("P group must be positive and divide the last axis")+ if scale_mode == "amax2_or_amax6_c8":+ values, winner = nvfp4_probability_quantize(+ x.float() * 8.0, group=group, scale_mode="amax2_or_amax6"+ )+ return values * 0.125, winner+ if scale_mode == "two_level":+ tile_size = min(128, x.shape[-1])+ if x.shape[-1] % tile_size:+ raise ValueError("two_level P needs complete 128-value tiles")+ tiles = x.float().reshape(*x.shape[:-1], -1, tile_size)+ peak = tiles.abs().amax(dim=-1, keepdim=True)+ outer = peak / (448.0 * 6.0)+ amplified = torch.where(peak > 0, tiles / peak.clamp_min(torch.finfo(torch.float32).tiny) * (448.0 * 6.0), 0.0)+ values, winner = nvfp4_probability_quantize(amplified, group=group, scale_mode="amax6")+ return (values * outer).reshape_as(x), winner.reshape(*x.shape[:-1], -1)+ groups = x.float().reshape(*x.shape[:-1], x.shape[-1] // group, group)+ amax = groups.abs().amax(dim=-1, keepdim=True)++ if scale_mode == "amax2_or_amax6":+ low_amax = amax <= P_SMALL_GROUP_AMAX+ reciprocal = torch.where(+ low_amax,+ torch.tensor(1.0 / 2.0, dtype=torch.float32, device=x.device),+ torch.tensor(1.0 / 6.0, dtype=torch.float32, device=x.device),+ )+ raw_scale = amax * reciprocal+ stored_scale = raw_scale.clamp(max=_E4M3_MAX).to(+ torch.float8_e4m3fn+ ).float()+ codes = round_e2m1(groups * (1.0 / (raw_scale + 1.0e-30)))+ values = codes * stored_scale+ winner = torch.where(+ low_amax.squeeze(-1),+ torch.full_like(amax.squeeze(-1), 3, dtype=torch.int64),+ torch.zeros_like(amax.squeeze(-1), dtype=torch.int64),+ )+ return values.reshape_as(x), winner++ best_values = best_error = best_winner = None+ divisors = probability_scale_candidates(scale_mode)+ for index, divisor in enumerate(divisors):+ reciprocal = torch.tensor(1.0 / divisor, dtype=torch.float32).item()+ raw_scale = amax * reciprocal+ stored_scale = raw_scale.clamp(max=_E4M3_MAX).to(+ torch.float8_e4m3fn+ ).float()+ codes = round_e2m1(groups * (1.0 / (raw_scale + 1.0e-30)))+ values = codes * stored_scale+ if len(divisors) == 1:+ winner = torch.zeros_like(amax.squeeze(-1), dtype=torch.int64)+ return values.reshape_as(x), winner+ error = _accumulate_squares(values - groups)+ if best_error is None:+ best_values = values+ best_error = error+ best_winner = torch.zeros_like(error, dtype=torch.int64)+ continue+ better = error < best_error+ best_values = torch.where(better, values, best_values)+ best_error = torch.where(better, error, best_error)+ best_winner = torch.where(+ better, torch.full_like(best_winner, index), best_winner+ )+ return best_values.reshape_as(x), best_winner.squeeze(-1)+++def probability_scale_candidates(scale_mode: str) -> tuple[float, ...]:+ """Candidate divisors supported by the ephemeral P quantizer.++ Cached K/V keep their existing scale-mode vocabulary. P additionally+ supports fixed interior E2M1 rungs and an offline oracle over those rungs;+ none of these modes changes the P payload or scale layout.+ """+ if scale_mode in _PROBABILITY_SCALE_MODES:+ return _PROBABILITY_SCALE_MODES[scale_mode]+ return candidates(scale_mode)+++def value_token_axis_round_trip(+ values: torch.Tensor,+ *,+ fp4_tokens: int,+ sink_tokens: int = 16,+ page_size: int = 128,+ scale_mode: str = "four_six",+ fmt: str = "nvfp4",+) -> torch.Tensor:+ """Round V with the cache's real token-axis grouping.++ ``values`` is ``[tokens, heads, head_dim]``. Each complete cache page is+ viewed as ``[heads, head_dim, tokens]`` before grouping, because token is+ the K dimension of the P×V MMA. The permanent sink and the incomplete+ recent tail remain BF16 and are restored after the page round trip.+ """+ if values.ndim != 3:+ raise ValueError(+ f"values must have shape [tokens,heads,dim], got {values.shape}"+ )+ tokens = values.shape[0]+ if (+ type(fp4_tokens) is not int+ or fp4_tokens < 0+ or fp4_tokens > tokens+ or fp4_tokens % page_size+ ):+ raise ValueError(+ "fp4_tokens must be a page-aligned integer inside the sequence"+ )+ if sink_tokens < 0 or sink_tokens > tokens:+ raise ValueError("sink_tokens must lie inside the sequence")++ rounded = values.float().clone()+ if fp4_tokens:+ pages = values[:fp4_tokens].reshape(+ -1, page_size, values.shape[1], values.shape[2]+ ).permute(0, 2, 3, 1)+ rounded_pages = fp4_round_trip(pages, fmt, scale_mode).permute(+ 0, 3, 1, 2+ )+ rounded[:fp4_tokens] = rounded_pages.reshape_as(+ rounded[:fp4_tokens]+ )+ rounded[:sink_tokens] = values[:sink_tokens].float()+ rounded[fp4_tokens:] = values[fp4_tokens:].float()+ return rounded+++def online_softmax_pv(+ scores: torch.Tensor,+ values: torch.Tensor,+ *,+ fp4_tokens: int,+ quantize_probabilities: bool,+ sink_tokens: int = 16,+ page_size: int = 128,+ softmax_scale: float | None = None,+ probability_format: str = "nvfp4",+ rescale_threshold: float = 0.0,+) -> torch.Tensor:+ """Accumulate P×V in the decode kernel's online block order.++ The incomplete BF16 tail is consumed first, complete FP4 pages from newest+ to oldest, and the permanent BF16 sink last. On FP4 pages, P is optionally+ rounded as ``probability_format``'s P operand+ (:func:`fp4_probability_round_trip`), NVFP4 with fixed ``amax/6``. The denominator+ always uses the unquantized exponentials, matching the kernel.++ ``rescale_threshold`` is the kernel's ``RESCALE_THRESHOLD``: a running+ maximum that would grow by at most that many powers of two is kept, so+ P is formed against the stale maximum. Zero always rescales.+ """+ if scores.ndim != 2 or values.ndim != 3:+ raise ValueError("scores must be [heads,tokens] and values [tokens,heads,dim]")+ heads, tokens = scores.shape+ if values.shape[:2] != (tokens, heads):+ raise ValueError(+ f"score/value shape mismatch: {scores.shape} versus {values.shape}"+ )+ if (+ type(fp4_tokens) is not int+ or fp4_tokens < 0+ or fp4_tokens > tokens+ or fp4_tokens % page_size+ ):+ raise ValueError(+ "fp4_tokens must be a page-aligned integer inside the sequence"+ )+ if sink_tokens < 0 or sink_tokens > tokens:+ raise ValueError("sink_tokens must lie inside the sequence")+ scale = float(softmax_scale or values.shape[-1] ** -0.5)+ scaled_scores = scores.float() * scale+ output = torch.zeros(+ heads, values.shape[-1], device=values.device, dtype=torch.float32+ )+ running_max = torch.full(+ (heads,), -torch.inf, device=values.device, dtype=torch.float32+ )+ running_sum = torch.zeros_like(running_max)++ def accumulate(+ block_scores: torch.Tensor,+ block_values: torch.Tensor,+ *,+ fp4_probability: bool,+ ) -> None:+ nonlocal output, running_max, running_sum+ block_max = torch.maximum(running_max, block_scores.amax(dim=-1))+ if rescale_threshold:+ growth = (block_max - running_max) * math.log2(math.e)+ block_max = torch.where(+ growth <= rescale_threshold, running_max, block_max+ )+ rescale = torch.exp(running_max - block_max)+ rescale = torch.where(+ torch.isfinite(rescale), rescale, torch.zeros_like(rescale)+ )+ finite_max = torch.where(+ torch.isfinite(block_max), block_max, torch.zeros_like(block_max)+ )+ probability = torch.exp(block_scores - finite_max[:, None])+ running_sum = running_sum * rescale + probability.sum(dim=-1)+ output = output * rescale[:, None]+ if fp4_probability:+ used_probability = fp4_probability_round_trip(+ probability, probability_format+ )+ else:+ used_probability = probability.to(torch.bfloat16).float()+ output = output + torch.einsum(+ "ht,thd->hd", used_probability, block_values.float()+ )+ running_max = block_max++ tail_start = sink_tokens if fp4_tokens == 0 else fp4_tokens+ if tail_start < tokens:+ accumulate(+ scaled_scores[:, tail_start:],+ values[tail_start:],+ fp4_probability=False,+ )++ for start in range(fp4_tokens - page_size, -1, -page_size):+ block_scores = scaled_scores[:, start : start + page_size].clone()+ if start < sink_tokens:+ invalid = torch.arange(+ start, start + page_size, device=scores.device+ ) < sink_tokens+ block_scores[:, invalid] = -torch.inf+ accumulate(+ block_scores,+ values[start : start + page_size],+ fp4_probability=quantize_probabilities,+ )++ if sink_tokens:+ accumulate(+ scaled_scores[:, :sink_tokens],+ values[:sink_tokens],+ fp4_probability=False,+ )+ return output / running_sum.clamp_min(1.0e-30)[:, None]+++def online_softmax_pv_factors(+ scores: torch.Tensor,+ bf16_values: torch.Tensor,+ fp4_values: torch.Tensor,+ *,+ fp4_tokens: int,+ sink_tokens: int = 16,+ page_size: int = 128,+ softmax_scale: float | None = None,+ probability_scale_mode: str = "amax6",+) -> dict[str, torch.Tensor]:+ """Isolate P and V errors in one production-order online-softmax pass.++ The four returned outputs share the exact same unquantized softmax+ denominator. Complete prefix pages use either BF16 exponentials or the+ kernel's fixed-amax/6 NVFP4 P operand, crossed with BF16 or token-axis+ NVFP4 V. Sink and incomplete-tail blocks remain BF16 in every arm.+ """+ if scores.ndim != 2 or bf16_values.ndim != 3 or fp4_values.ndim != 3:+ raise ValueError(+ "scores must be [heads,tokens] and values [tokens,heads,dim]"+ )+ heads, tokens = scores.shape+ expected = (tokens, heads)+ if bf16_values.shape[:2] != expected or fp4_values.shape != bf16_values.shape:+ raise ValueError("score/value shapes do not agree")+ if (+ type(fp4_tokens) is not int+ or fp4_tokens < 0+ or fp4_tokens > tokens+ or fp4_tokens % page_size+ ):+ raise ValueError(+ "fp4_tokens must be a page-aligned integer inside the sequence"+ )+ if sink_tokens < 0 or sink_tokens > tokens:+ raise ValueError("sink_tokens must lie inside the sequence")++ scale = float(softmax_scale or bf16_values.shape[-1] ** -0.5)+ scaled_scores = scores.float() * scale+ outputs = {+ name: torch.zeros(+ heads,+ bf16_values.shape[-1],+ device=bf16_values.device,+ dtype=torch.float32,+ )+ for name in ("reference", "p_only", "v_only", "p_and_v")+ }+ running_max = torch.full(+ (heads,), -torch.inf, device=scores.device, dtype=torch.float32+ )+ running_sum = torch.zeros_like(running_max)++ def accumulate(start: int, end: int, *, fp4_block: bool) -> None:+ nonlocal running_max, running_sum+ block_scores = scaled_scores[:, start:end]+ if fp4_block and start < sink_tokens:+ block_scores = block_scores.clone()+ invalid = torch.arange(start, end, device=scores.device) < sink_tokens+ block_scores[:, invalid] = -torch.inf+ block_max = torch.maximum(running_max, block_scores.amax(dim=-1))+ rescale = torch.exp(running_max - block_max)+ rescale = torch.where(+ torch.isfinite(rescale), rescale, torch.zeros_like(rescale)+ )+ finite_max = torch.where(+ torch.isfinite(block_max), block_max, torch.zeros_like(block_max)+ )+ probability = torch.exp(block_scores - finite_max[:, None])+ running_sum = running_sum * rescale + probability.sum(dim=-1)+ for name in outputs:+ outputs[name] *= rescale[:, None]++ bf16_probability = probability.to(torch.bfloat16).float()+ fp4_probability = (+ nvfp4_probability_quantize(+ probability,+ group=16,+ scale_mode=probability_scale_mode,+ )[0]+ if fp4_block+ else bf16_probability+ )+ bf16_block = bf16_values[start:end].float()+ fp4_block_values = (+ fp4_values[start:end].float() if fp4_block else bf16_block+ )+ outputs["reference"] += torch.einsum(+ "ht,thd->hd", bf16_probability, bf16_block+ )+ outputs["p_only"] += torch.einsum(+ "ht,thd->hd", fp4_probability, bf16_block+ )+ outputs["v_only"] += torch.einsum(+ "ht,thd->hd", bf16_probability, fp4_block_values+ )+ outputs["p_and_v"] += torch.einsum(+ "ht,thd->hd", fp4_probability, fp4_block_values+ )+ running_max = block_max++ tail_start = sink_tokens if fp4_tokens == 0 else fp4_tokens+ if tail_start < tokens:+ accumulate(tail_start, tokens, fp4_block=False)+ for start in range(fp4_tokens - page_size, -1, -page_size):+ accumulate(start, start + page_size, fp4_block=True)+ if sink_tokens:+ accumulate(0, sink_tokens, fp4_block=False)++ denominator = running_sum.clamp_min(1.0e-30)[:, None]+ return {name: output / denominator for name, output in outputs.items()}+++def online_probability_scale_statistics(+ scores: torch.Tensor,+ *,+ fp4_tokens: int,+ head_dim: int,+ sink_tokens: int = 16,+ page_size: int = 128,+ probability_multiplier: float = 1.0,+) -> dict:+ """Aggregate exact P-scale errors by the E4M3 code of each group maximum.++ This vectorizes the same reverse-page online-softmax order used by+ :func:`online_softmax_pv_factors`. It is an offline diagnostic: the+ returned code buckets are used to ask whether a cheap amax-code lookup can+ approximate the expensive four-round-trip per-group oracle.+ """+ if scores.ndim != 2:+ raise ValueError("scores must have shape [heads, tokens]")+ heads, tokens = scores.shape+ if fp4_tokens <= 0 or fp4_tokens > tokens or fp4_tokens % page_size:+ raise ValueError("fp4_tokens must be positive, page aligned, and in range")+ if page_size % 16:+ raise ValueError("page size must be divisible by the P group size")+ if not 0 <= sink_tokens <= fp4_tokens:+ raise ValueError("sink tokens must lie inside the FP4 prefix")+ if not math.isfinite(probability_multiplier) or not 0 < probability_multiplier <= 448:+ raise ValueError("probability_multiplier must be finite and between 0 and 448")++ scaled = scores.float() * float(head_dim**-0.5)+ pages = scaled[:, :fp4_tokens].reshape(heads, -1, page_size).clone()+ if sink_tokens:+ pages[:, 0, :sink_tokens] = -torch.inf+ page_max = pages.amax(dim=-1)+ suffix_max = torch.flip(+ torch.cummax(torch.flip(page_max, dims=(1,)), dim=1).values,+ dims=(1,),+ )+ if fp4_tokens < tokens:+ tail_max = scaled[:, fp4_tokens:].amax(dim=-1, keepdim=True)+ suffix_max = torch.maximum(suffix_max, tail_max)+ finite_max = torch.where(+ torch.isfinite(suffix_max), suffix_max, torch.zeros_like(suffix_max)+ )+ probability = torch.exp(pages - finite_max[..., None])+ probability = torch.where(+ torch.isfinite(pages), probability, torch.zeros_like(probability)+ )+ groups = (probability * probability_multiplier).reshape(-1, 16)+ maximum = groups.amax(dim=-1, keepdim=True)+ maximum_code = maximum.squeeze(-1).to(torch.float8_e4m3fn).view(torch.uint8)++ divisors = probability_scale_candidates("e2m1_max_grid")+ errors = []+ for divisor in divisors:+ raw_scale = maximum * float(1.0 / divisor)+ stored_scale = raw_scale.clamp(max=_E4M3_MAX).to(+ torch.float8_e4m3fn+ ).float()+ codes = round_e2m1(groups / raw_scale.clamp_min(1.0e-30))+ errors.append((codes * stored_scale - groups).square().sum(dim=-1))+ error = torch.stack(errors, dim=-1)+ winner = error.argmin(dim=-1)+ oracle_error = error.gather(-1, winner[:, None]).squeeze(-1)++ bucket_count = torch.bincount(maximum_code.long(), minlength=256)+ winner_count = torch.bincount(+ maximum_code.long() * len(divisors) + winner,+ minlength=256 * len(divisors),+ ).reshape(256, len(divisors))+ error_sum = torch.zeros(+ 256, len(divisors), device=error.device, dtype=torch.float64+ )+ error_sum.scatter_add_(+ 0,+ maximum_code.long()[:, None].expand_as(error),+ error.double(),+ )+ oracle_sum = torch.zeros(256, device=error.device, dtype=torch.float64)+ oracle_sum.scatter_add_(0, maximum_code.long(), oracle_error.double())++ counts = bucket_count.cpu().tolist()+ winners = winner_count.cpu().tolist()+ sums = error_sum.cpu().tolist()+ oracle = oracle_sum.cpu().tolist()+ by_code = {+ str(code): {+ "groups": int(counts[code]),+ "winner_counts": {+ str(int(divisor)): int(winners[code][index])+ for index, divisor in enumerate(divisors)+ },+ "error_sum": {+ str(int(divisor)): float(sums[code][index])+ for index, divisor in enumerate(divisors)+ },+ "oracle_error_sum": float(oracle[code]),+ }+ for code in range(256)+ if counts[code]+ }+ return {+ "group_count": int(groups.shape[0]),+ "divisors": [int(value) for value in divisors],+ "winner_counts": {+ str(int(divisor)): int((winner == index).sum().item())+ for index, divisor in enumerate(divisors)+ },+ "error_sum": {+ str(int(divisor)): float(error[:, index].double().sum().item())+ for index, divisor in enumerate(divisors)+ },+ "oracle_error_sum": float(oracle_error.double().sum().item()),+ "by_amax_e4m3_code": by_code,+ }diff --git a/src/nvfp4_decode_kernel/scale_mode.py b/src/nvfp4_decode_kernel/scale_mode.pynew file mode 100644index 0000000..99b1ffd--- /dev/null+++ b/src/nvfp4_decode_kernel/scale_mode.py@@ -0,0 +1,76 @@+"""The block-scale candidate sets the quantization entries share.++A group's scale is its largest magnitude divided by one of these, rounded to+E4M3. One candidate is what every version of this cache has stored; the others+exist because ``amax/6``'s widest rung, the gap between 4 and 6, is four times+the grid's own step and swallows whatever second-largest element lands in it.++Order inside a tuple is tie-break order, earliest wins. That makes ``amax/6``+the answer whenever nothing beats it, so a larger set is a strict improvement+on a smaller one rather than a different answer, and it makes the extra+candidates byte-identical to the set without them on every group they do not+win. Both properties are what lets the three modes be compared as one variable.+"""++from __future__ import annotations+++SCALE_MODES: dict[str, tuple[float, ...]] = {+ "amax6": (6.0,),+ "four_six": (6.0, 4.0),+ "four_six_sixhalf": (6.0, 4.0, 6.5),+}++DEFAULT_SCALE_MODE = "amax6"+++def parse_scale_mode_spec(spec: str) -> str | dict[str, str]:+ """Parse a CLI scale mode without importing the attention implementation."""+ if "=" not in spec:+ return spec+ modes = dict(part.split("=", 1) for part in spec.split(",") if part)+ fields = {"key", "value", "query"}+ if set(modes) != fields:+ raise ValueError(+ f"a per-tensor scale mode needs exactly {sorted(fields)}, got {sorted(modes)}"+ )+ return modes++# MXFP4 modes: 32 values per power-of-two (E8M0) scale. The value is the+# exponent rule ``reference.mxfp4_quantize`` and ``_block_scale`` apply.+MX_SCALE_MODES: dict[str, str] = {+ "mxfp4_ocp": "ocp",+ "mxfp4_round_up": "round_up",+ "mxfp4_search": "search",+}+++def scale_format(scale_mode: str) -> str:+ """``"mxfp4"`` for an MX mode, ``"nvfp4"`` for a candidate set."""+ if scale_mode in MX_SCALE_MODES:+ return "mxfp4"+ candidates(scale_mode)+ return "nvfp4"+++def sf_vec_size(scale_mode: str) -> int:+ """How many values share one scale under ``scale_mode``."""+ return 32 if scale_format(scale_mode) == "mxfp4" else 16+++def candidates(scale_mode: str) -> tuple[float, ...]:+ """The divisors an NVFP4 ``scale_mode`` names, in tie-break order."""+ try:+ return SCALE_MODES[scale_mode]+ except KeyError:+ raise ValueError(+ f"unknown scale_mode {scale_mode!r}, expected one of "+ f"{sorted(SCALE_MODES) + sorted(MX_SCALE_MODES)}"+ ) from None+++def scale_rule(scale_mode: str) -> tuple[float, ...] | str:+ """What a quantizer specializes on: NVFP4 divisors, or an MX exponent rule."""+ if scale_mode in MX_SCALE_MODES:+ return MX_SCALE_MODES[scale_mode]+ return candidates(scale_mode)diff --git a/src/nvfp4_decode_kernel/split_k_combine.py b/src/nvfp4_decode_kernel/split_k_combine.pynew file mode 100644index 0000000..cb3a9cf--- /dev/null+++ b/src/nvfp4_decode_kernel/split_k_combine.py@@ -0,0 +1,136 @@+"""LSE-weighted reduction of split-K decode partials into the BF16 output."""++import math++import cuda.bindings.driver as cuda+import cutlass+import cutlass.cute as cute+from cutlass import Float32++LOG2_E = math.log2(math.e)+++class SplitKCombine:+ """One CTA per decode row and query head, one thread per head-dim element.++ A thread loads every split's LSE and partial output for its element at+ once, so the reduction waits on L2 once whatever the split count. Walking+ the splits through a short pipeline instead costs a round trip every few+ splits, which at batch 1, where few CTAs each carry 32 or more splits, was+ most of the combine's time.++ A split that was handed no work wrote LSE -inf and left its partial output+ as the scratch held it, so a partial is summed only under a positive+ weight: NaN in an unwritten slot never reaches the output.+ """++ def __init__(self, head_dim: int, num_splits: int):+ self.head_dim = head_dim+ self.num_splits = num_splits++ @cute.jit+ def __call__(+ self,+ mO_partial: cute.Tensor,+ mLSE_partial: cute.Tensor,+ mO: cute.Tensor,+ stream: cuda.CUstream,+ ):+ # mO_partial (splits, rows, 1, heads, head_dim), mLSE_partial+ # (splits, rows, 1, heads), mO (rows, 1, heads, head_dim).+ self.kernel(mO_partial, mLSE_partial, mO).launch(+ grid=(mO.shape[2], mO.shape[0], 1),+ block=(self.head_dim, 1, 1),+ stream=stream,+ )++ @cute.kernel+ def kernel(+ self,+ mO_partial: cute.Tensor,+ mLSE_partial: cute.Tensor,+ mO: cute.Tensor,+ ):+ dim, _, _ = cute.arch.thread_idx()+ head, row, _ = cute.arch.block_idx()+ weight = cute.make_rmem_tensor(cute.make_layout(self.num_splits), Float32)+ partial = cute.make_rmem_tensor(cute.make_layout(self.num_splits), Float32)+ for s in cutlass.range_constexpr(self.num_splits):+ weight[s] = mLSE_partial[s, row, 0, head]+ partial[s] = mO_partial[s, row, 0, head, dim]++ lse_max = weight[0]+ for s in cutlass.range_constexpr(1, self.num_splits):+ lse_max = cute.arch.fmax(lse_max, weight[s])+ # Every split empty: every weight is then exp2(-inf) = 0.+ lse_max = 0.0 if lse_max == -Float32.inf else lse_max+ weight_sum = Float32(0.0)+ for s in cutlass.range_constexpr(self.num_splits):+ weight[s] = cute.math.exp2(+ weight[s] * LOG2_E - (lse_max * LOG2_E), fastmath=True+ )+ weight_sum += weight[s]+ inverse = (+ 0.0 if (weight_sum == 0.0 or weight_sum != weight_sum) else 1.0 / weight_sum+ )++ # Split order, so a given set of partials always sums to the same bits.+ out = Float32(0.0)+ for s in cutlass.range_constexpr(self.num_splits):+ scale = weight[s] * inverse+ if scale > 0.0:+ out = out + scale * partial[s]+ mO[row, 0, head, dim] = cutlass.BFloat16(out)+++def combine_split_partials(out_partial, lse_partial, out=None):+ """Reduce ``out_partial`` ``(splits, rows, 1, heads, head_dim)`` FP32 under+ ``lse_partial`` ``(splits, rows, 1, heads)`` FP32 into BF16+ ``(rows, 1, heads, head_dim)``, written to ``out`` when given."""+ import torch+ from cutlass.cute.runtime import from_dlpack++ num_splits, rows, seqlen, heads, head_dim = out_partial.shape+ if seqlen != 1:+ raise ValueError(f"the split-K combine serves one query position, got {seqlen}")+ if out is None:+ out = torch.empty(+ rows, seqlen, heads, head_dim,+ dtype=torch.bfloat16, device=out_partial.device,+ )+ # One kernel per split count: every split is a register and an exponential+ # per thread, so a padded tier would spend that on slots no split fills.+ # Serving launches one count per captured width.+ key = (head_dim, num_splits)++ def to_cute(tensor, leading_dim):+ return from_dlpack(+ tensor.detach(), assumed_align=16, enable_tvm_ffi=True+ ).mark_layout_dynamic(leading_dim=leading_dim)++ cute_args = (+ to_cute(out_partial, 4),+ to_cute(lse_partial, 3),+ to_cute(out, 3),+ )+ compiled = combine_split_partials.compile_cache.get(key)+ if compiled is None:+ compiled = cute.compile(+ SplitKCombine(head_dim, num_splits),+ *cute_args,+ cute.runtime.make_fake_stream(),+ options="--enable-tvm-ffi",+ )+ combine_split_partials.compile_cache[key] = compiled+ # Nothing orders this against the attention kernel that wrote the partials+ # except sharing its stream, and the TVM FFI environment's is the default+ # one. That went unnoticed until a host that keeps its work off the default+ # stream — vLLM does — turned the read into a race against the write.+ compiled(+ *cute_args,+ cuda.CUstream(torch.cuda.current_stream(out_partial.device).cuda_stream),+ )+ return out+++combine_split_partials.compile_cache = {}diff --git a/src/nvfp4_vllm/__init__.py b/src/nvfp4_vllm/__init__.pynew file mode 100644index 0000000..63b1f35--- /dev/null+++ b/src/nvfp4_vllm/__init__.py@@ -0,0 +1,47 @@+"""Out-of-tree NVFP4 KV cache backend for vLLM V1."""+++def register() -> None:+ """Entry point for the ``vllm.general_plugins`` group."""+ from vllm.v1.attention.backends.registry import (+ AttentionBackendEnum,+ register_backend,+ )++ register_backend(+ AttentionBackendEnum.CUSTOM,+ "nvfp4_vllm.backend.NVFP4Backend",+ )+++# All three are imported lazily so that ``register`` stays free of torch and+# vLLM, which the plugin entry point is loaded too early to depend on.+++def build_llm(model, **kwargs):+ """See ``nvfp4_vllm.engine.build_llm``."""+ from .engine import build_llm as _build++ return _build(model, **kwargs)+++def calibrate_key_smoothing(llm, **kwargs):+ """Calibrate static key smoothing, optionally fitting the probability threshold."""+ from .calibrate import calibrate_key_shift as _calibrate++ return _calibrate(llm, **kwargs)+++def set_key_smoothing_enabled(llm, enabled: bool):+ """Enable or disable an engine's calibrated static key smoothing."""+ from .calibrate import set_key_shift_enabled as _set++ return _set(llm, enabled)+++__all__ = [+ "register",+ "build_llm",+ "calibrate_key_smoothing",+ "set_key_smoothing_enabled",+]diff --git a/src/nvfp4_vllm/backend.py b/src/nvfp4_vllm/backend.pynew file mode 100644index 0000000..7abf0a8--- /dev/null+++ b/src/nvfp4_vllm/backend.py@@ -0,0 +1,151 @@+"""Attention backend that declares the NVFP4 KV cache page layout to vLLM."""++from __future__ import annotations++from typing import TYPE_CHECKING, ClassVar++import torch+from vllm.config import get_current_vllm_config_or_none+from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend++from .guards import NVFP4++if TYPE_CHECKING:+ from vllm.platforms.interface import DeviceCapability+++PAGE_SIZE = 128+HEAD_SIZE = 128+SM100 = 10+++def _cache_dtype() -> str | None:+ """The configured ``kv_cache_dtype``, or None outside an engine."""+ config = get_current_vllm_config_or_none()+ if config is None or config.cache_config is None:+ return None+ return config.cache_config.cache_dtype+++def nvfp4_page_size_bytes(+ block_size: int, num_kv_heads: int, head_size: int+) -> int:+ """Bytes one K+V page occupies for a single layer.++ E2M1 data packs two values per byte and each group of 16 values carries one+ E4M3 scale byte. vLLM budgets the cache from the same expression in+ ``AttentionSpec.real_page_size_bytes``, so the two must agree exactly or the+ allocation will not match the shape.+ """+ packed_head_dim = head_size // 2 + head_size // 16+ return 2 * block_size * num_kv_heads * packed_head_dim+++class NVFP4Backend(FlashAttentionBackend):+ """Paged NVFP4 KV cache backend.++ Reuses FlashAttention metadata and BF16 prefill. Decode and KV-cache+ writes use the custom FP4 kernels.+ """++ supported_kv_cache_dtypes: ClassVar[list[str]] = [+ *FlashAttentionBackend.supported_kv_cache_dtypes,+ NVFP4,+ ]++ @staticmethod+ def get_name() -> str:+ # vLLM resolves the enum via ``AttentionBackendEnum[get_name()]``.+ return "CUSTOM"++ @staticmethod+ def get_impl_cls() -> type:+ from .impl import NVFP4Impl++ return NVFP4Impl++ @staticmethod+ def get_builder_cls() -> type:+ from .builder import NVFP4MetadataBuilder++ return NVFP4MetadataBuilder++ @staticmethod+ def get_supported_kernel_block_sizes() -> list[int]:+ # V is quantized along the token axis a full page at a time, so a page+ # cannot be split or partially filled.+ return [PAGE_SIZE]++ @classmethod+ def supports_block_size(cls, block_size: int | None) -> bool:+ # Stricter than the inherited "multiple of 128": vLLM would otherwise+ # accept a 256-token cache block and split it into two kernel blocks.+ return block_size is None or block_size == PAGE_SIZE++ @classmethod+ def supports_compute_capability(cls, capability: DeviceCapability) -> bool:+ return capability.major == SM100++ @classmethod+ def supports_kv_connector(cls) -> bool:+ # A connector moves pages by address and size, which does not describe+ # the packed FP4 layout.+ return False++ @classmethod+ def supports_combination(+ cls,+ head_size: int,+ dtype: torch.dtype,+ kv_cache_dtype: str | None,+ block_size: int | None,+ use_mla: bool,+ has_sink: bool,+ use_sparse: bool,+ use_mm_prefix: bool,+ device_capability: DeviceCapability,+ ) -> str | None:+ # Only meaningful for the FP4 path; with any other cache dtype this+ # backend is a FlashAttention pass-through. Both head_size and+ # kv_cache_dtype are part of the selector's cache key, so a verdict+ # conditioned on them cannot be reused for a different combination.+ if kv_cache_dtype == NVFP4 and head_size != HEAD_SIZE:+ return (+ f"NVFP4 decode is compiled for head_size {HEAD_SIZE}, "+ f"got {head_size}"+ )+ return None++ @staticmethod+ def get_kv_cache_shape(+ num_blocks: int,+ block_size: int,+ num_kv_heads: int,+ head_size: int,+ cache_dtype_str: str = "auto",+ ) -> tuple[int, ...]:+ if cache_dtype_str != NVFP4:+ return FlashAttentionBackend.get_kv_cache_shape(+ num_blocks,+ block_size,+ num_kv_heads,+ head_size,+ cache_dtype_str=cache_dtype_str,+ )+ # A flat per-page byte view, committing to nothing about how the four+ # regions (K data, K scales, V data, V scales) sit inside it.+ page_bytes = nvfp4_page_size_bytes(block_size, num_kv_heads, head_size)+ return (num_blocks, page_bytes)++ @staticmethod+ def get_kv_cache_stride_order(+ include_num_layers_dimension: bool = False,+ ) -> tuple[int, ...]:+ # The permutation must have the same rank as the shape above, and vLLM+ # does not pass the cache dtype here. Declining makes the caller fall+ # back to the identity order, which is what a flat byte page needs.+ if _cache_dtype() == NVFP4:+ raise NotImplementedError+ return FlashAttentionBackend.get_kv_cache_stride_order(+ include_num_layers_dimension+ )diff --git a/src/nvfp4_vllm/builder.py b/src/nvfp4_vllm/builder.pynew file mode 100644index 0000000..7d3b9a0--- /dev/null+++ b/src/nvfp4_vllm/builder.py@@ -0,0 +1,301 @@+"""Assign BF16 tail slots once per scheduler step, before attention layers run.++The control plane's counter tracks scheduler steps. Microbatching would build+metadata more than once per step and is rejected by ``guards``.++CUDA graph capture uses null-block rows, which receive no slots. The first+real ``build`` after capture resets the table. Startup profiling also uses+null-block rows; abandoned warmup slots are reclaimed lazily.+"""++from __future__ import annotations++import os++import torch+from vllm.config import VllmConfig+from vllm.v1.attention.backend import (+ AttentionCGSupport,+ CommonAttentionMetadata,+)+from vllm.v1.attention.backends.flash_attn import (+ FlashAttentionMetadata,+ FlashAttentionMetadataBuilder,+)+from vllm.v1.attention.backends.utils import split_decodes_and_prefills+from vllm.v1.kv_cache_interface import AttentionSpec++from .control import PAGE_SIZE, ControlPlane, split_sequence_length+from .guards import NVFP4, check_supported+from .metadata import NVFP4Metadata+from .write import PageWorkTable+++# Read the control plane's sticky error word after each step for debugging.+# Disabled by default because reading it synchronizes the host and device.+DEBUG_ENV = "NVFP4_DEBUG"++# The largest power-of-two split count. A count this high already leaves an 8k+# context two pages per split; filling the SMs it leaves idle can take a count+# up to one short of twice it.+MAX_DECODE_SPLITS = 32+++def decode_num_splits(rows: int, num_kv_heads: int, sms: int) -> int:+ """Choose key partitions from batch width, KV heads and available SMs.++ Start with a power of two, then fill otherwise idle SMs while reserving+ one CTA per row and KV head for the BF16 tail and sink. Stay below the+ next power of two to retain the same combine tier.++ The choice uses no sequence lengths, so it remains valid when a captured+ CUDA graph is replayed at a different context length.+ """+ if rows < 1:+ return 1+ ctas_per_split = rows * num_kv_heads+ budget = min(sms // ctas_per_split, MAX_DECODE_SPLITS)+ splits = 1+ while splits * 2 <= budget:+ splits *= 2+ filled = sms // ctas_per_split - 1+ return filled if splits < filled < 2 * splits else splits+++def decode_split(+ common_attn_metadata: CommonAttentionMetadata,+) -> tuple[int, int]:+ """Where this step's decode prefix ends, in rows and in tokens.++ The one place this module looks at a batch on the host, and it is allowed+ to because ``query_start_loc_cpu`` is something the model runner already+ materialized, so reading it costs no synchronization. What it guards is the+ premise the whole read path rests on — that the reordering actually+ happened. Without it, a vLLM that stopped reordering would have the decode+ kernel attend to prefill rows' caches with decode rows' queries, which is+ wrong everywhere and loud nowhere.++ A batch dispatched to a full CUDA graph is padded out to the captured+ width with rows carrying no token at all. Those rows stay inside the+ returned prefix, because the kernels have to keep the shape the graph was+ captured at, and the control plane gives them no slot and no length. They+ are only ever a suffix: vLLM pads a uniform decode batch and nothing else,+ so a zero-length row among the real ones would mean something other than+ padding put it there.+ """+ num_decodes, _, num_decode_tokens, _ = split_decodes_and_prefills(+ common_attn_metadata, decode_threshold=1+ )+ query_start_loc = common_attn_metadata.query_start_loc_cpu+ query_lens = query_start_loc[1 : common_attn_metadata.num_reqs + 1] - (+ query_start_loc[: common_attn_metadata.num_reqs]+ )+ decode_lens = query_lens[:num_decodes]+ # Where the padding starts, if there is any. Comparing only below it also+ # decides that the zero-length rows are contiguous at the end, since a zero+ # anywhere earlier falls inside the compared range.+ scheduled = int((decode_lens > 0).sum())+ # The last condition is the one that can fire. vLLM counts the prefix by+ # scanning one-token rows from the front, so an unsorted batch gives a+ # prefix that is too short rather than a wrong one, and what gives it away+ # is a one-token row left behind among the prefills.+ if not bool((decode_lens[:scheduled] == 1).all()) or not bool(+ (query_lens[num_decodes:] > 1).all()+ ):+ raise ValueError(+ "the NVFP4 read path needs one-token requests at the front of the "+ f"batch, but query lengths are {query_lens.tolist()} with a decode "+ f"prefix of {num_decodes}"+ )+ return num_decodes, num_decode_tokens+++class NVFP4MetadataBuilder(FlashAttentionMetadataBuilder):+ """FlashAttention's builder, extended with this step's slot assignment.++ A pass-through under any other cache dtype: the backend is selected per+ engine, not per cache dtype, so this class is also what a BF16 run gets.++ ``supports_update_block_table`` stays inherited. It is what keeps a model+ with several KV cache groups to one ``build()`` per step, and the base+ implementation shallow-copies the metadata object, so the slot fields+ below survive the copy.++ Under NVFP4 it also asks vLLM to sort decode requests to the front of the+ batch; see ``decode_split``.+ """++ @classmethod+ def get_cudagraph_support(+ cls,+ vllm_config: VllmConfig,+ kv_cache_spec: AttentionSpec,+ ) -> AttentionCGSupport:+ """How much of a batch this backend can have captured.++ FlashAttention answers ``ALWAYS`` on FA3, which claims a mixed+ prefill+decode batch can go into one graph. That is true of it and not+ of us: an NVFP4 prefill runs varlen FlashAttention over a token count+ that changes every step, so only the uniform one-token decode batches+ can be frozen. vLLM reads this before any builder is constructed, so+ the cache dtype has to come from the configuration rather than from+ ``self.nvfp4``.+ """+ cache_config = vllm_config.cache_config+ if cache_config is None or cache_config.cache_dtype != NVFP4:+ return super().get_cudagraph_support(vllm_config, kv_cache_spec)+ return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE++ def __init__(+ self,+ kv_cache_spec: AttentionSpec,+ layer_names: list[str],+ vllm_config: VllmConfig,+ device: torch.device,+ ) -> None:+ super().__init__(kv_cache_spec, layer_names, vllm_config, device)++ cache_config = vllm_config.cache_config+ self.nvfp4 = cache_config is not None and cache_config.cache_dtype == NVFP4+ # Raised once capture has asked for metadata, and read by the first+ # ``build`` that is a real step again. ``capturing`` distinguishes the+ # builds capture drives from that step, since both reach ``build``.+ self.built_for_capture = False+ self.capturing = False+ if not self.nvfp4:+ self.plane = None+ return++ # Read once. Per step this has to be a bool test, not a dict lookup.+ self.debug = os.environ.get(DEBUG_ENV) == "1"++ # Ask the model runner to move one-token requests to the front of the+ # batch. That makes the decode rows a contiguous prefix, so the decode+ # kernel takes slices of this step's arrays instead of a compacted copy+ # of them, and its output is a slice of vLLM's. Left unset above, so a+ # BF16 run through this backend keeps FlashAttention's batch order.+ self._init_reorder_batch_threshold(1)++ # Already enforced when the layers were constructed. Repeated here so a+ # builder is safe to construct on its own, and so the control plane is+ # never sized from a configuration it cannot serve.+ check_supported(vllm_config)++ scheduler_config = vllm_config.scheduler_config+ self.plane = ControlPlane(+ max_num_seqs=scheduler_config.max_num_seqs,+ max_num_batched_tokens=scheduler_config.max_num_batched_tokens,+ device=device,+ )+ # Which full pages this step writes depends only on the batch, so it is+ # resolved here alongside the slot assignment rather than once per+ # layer.+ self.work_table = PageWorkTable(+ max_num_seqs=scheduler_config.max_num_seqs,+ max_num_batched_tokens=scheduler_config.max_num_batched_tokens,+ device=device,+ )+ # Asked once: the split count is chosen per step, and a device+ # property query on that path would be a host call per step.+ self.num_kv_heads = kv_cache_spec.num_kv_heads+ self.sms = torch.cuda.get_device_properties(device).multi_processor_count++ def build_for_cudagraph_capture(+ self, common_attn_metadata: CommonAttentionMetadata+ ) -> FlashAttentionMetadata:+ """Metadata for a batch being recorded into a graph, not served.++ Nothing here differs from a normal build; the point is the record that+ it happened, so that ``build`` can throw away whatever capture left in+ the slot table. Capture is expected to leave nothing — every row of a+ capture batch keys on the null block and is not live — but this is the+ one moment in an engine's life when no request exists to lose, so the+ cheap thing to do is not rely on that.+ The base implementation routes straight back into ``build``, so the+ flag is raised only once that has returned. Raising it first would+ have ``build`` consume it on the way through and leave it low when+ capture ends, which is the one moment it has to be high.+ """+ self.capturing = True+ try:+ return super().build_for_cudagraph_capture(common_attn_metadata)+ finally:+ self.capturing = False+ if self.nvfp4:+ self.built_for_capture = True++ def build(+ self,+ common_prefix_len: int,+ common_attn_metadata: CommonAttentionMetadata,+ fast_build: bool = False,+ ) -> FlashAttentionMetadata:+ base = super().build(common_prefix_len, common_attn_metadata, fast_build)+ if not self.nvfp4:+ return base++ if self.built_for_capture and not self.capturing:+ # First step after capture, and only that one: the builds capture+ # itself makes arrive through here too, and clearing the table+ # between them would just be undone by the next one. The table+ # goes back to the state it had before the engine warmed up,+ # including the step counter each captured size advanced.+ self.built_for_capture = False+ self.plane.reset()++ # seq_lens is read as num_computed_tokens + num_scheduled_tokens, and+ # the FP4/BF16 page split is derived from it, so the other reading would+ # move every page boundary by one step. Nothing asserts it here: the+ # host copies the model runner computes it from are not on+ # CommonAttentionMetadata, and the reading is self-reporting anyway. If+ # it ever flips, a fresh prefill arrives with a zero length, is taken+ # for padding, gets no slot, and returns one step later with history it+ # cannot place, which is ERR_SLOT_LOST.+ outputs = self.plane.prepare(+ block_table=common_attn_metadata.block_table_tensor,+ seq_lens=common_attn_metadata.seq_lens,+ query_start_loc=common_attn_metadata.query_start_loc,+ num_reqs=common_attn_metadata.num_reqs,+ num_actual_tokens=common_attn_metadata.num_actual_tokens,+ )++ if self.debug:+ # Placed after prepare rather than before, so the step that broke+ # the invariant is the step that raises. The flags are sticky, so+ # every later step would raise too, but the first one is the one+ # with a batch worth looking at.+ self.plane.raise_for_errors()++ source_tokens, destination_pages = self.work_table.build(+ query_start_loc=common_attn_metadata.query_start_loc,+ seqused_fp4=outputs.seqused_fp4,+ row_to_slot=outputs.row_to_slot,+ block_table=common_attn_metadata.block_table_tensor,+ num_reqs=common_attn_metadata.num_reqs,+ max_query_len=common_attn_metadata.max_query_len,+ )++ decode_rows, decode_tokens = decode_split(common_attn_metadata)+ query_start_loc = common_attn_metadata.query_start_loc+ if decode_tokens:+ # Only a mixed batch pays for this. A step of nothing but prefills+ # already starts at zero, which is the shape a prefill step+ # normally has.+ query_start_loc = query_start_loc[decode_rows:] - decode_tokens++ _, max_fp4, _ = split_sequence_length(+ common_attn_metadata.max_seq_len+ )+ return NVFP4Metadata.from_flash(+ base,+ outputs,+ source_tokens=source_tokens,+ destination_pages=destination_pages,+ decode_prefix_rows=decode_rows,+ decode_prefix_tokens=decode_tokens,+ prefill_query_start_loc=query_start_loc,+ decode_page_columns=max(1, max_fp4 // PAGE_SIZE),+ decode_num_splits=decode_num_splits(+ decode_rows, self.num_kv_heads, self.sms+ ),+ )diff --git a/src/nvfp4_vllm/calibrate.py b/src/nvfp4_vllm/calibrate.pynew file mode 100644index 0000000..cfced3a--- /dev/null+++ b/src/nvfp4_vllm/calibrate.py@@ -0,0 +1,337 @@+"""Calibrate static key smoothing and optionally the probability threshold.++Model-generated text is tiled across the context window to sample post-RoPE+keys at representative positions. Subtracting their per-channel mean reduces+quantization error without changing softmax attention.++Calibration completes before serving so every cached key uses the same mean.+Named ``collective_rpc`` methods run calibration in each worker and save the+sidecar there, avoiding tensor serialization through the engine client.+"""++from __future__ import annotations++from collections import Counter+from typing import Any++import torch+++# vLLM mixes this into the worker, so ``collective_rpc`` can name its methods.+# An engine that wants to calibrate must be built with+# ``worker_extension_cls="nvfp4_vllm.calibrate.NVFP4WorkerExtension"``.+WORKER_EXTENSION_CLS = "nvfp4_vllm.calibrate.NVFP4WorkerExtension"+++SEEDS = (+ "Explain how a hash table resolves collisions, and when each strategy "+ "is preferable.",+ "Summarise the causes of the 1929 financial crash for a reader who "+ "knows no economics.",+ "Write a short story about a lighthouse keeper who stops receiving "+ "supply ships.",+ "Derive the quadratic formula from completing the square, showing every "+ "step.",+ "Describe how photosynthesis converts light into chemical energy, at the "+ "level of a first-year undergraduate.",+ "Compare pessimistic and optimistic concurrency control in databases.",+ "Translate the idea of technical debt into terms a non-engineer manager "+ "would act on.",+ "Walk through debugging a service that is slow only under high load.",+)+TOKENS_PER_SEED = 256+# Calibration spans this many positions, clamped to the engine's window.+CONTEXT_TOKENS = 32_768+++def _runtimes(model: torch.nn.Module) -> list[Any]:+ """The distinct LayerRuntime objects this model's attention layers share."""+ from .impl import NVFP4Impl++ seen: list[Any] = []+ for module in model.modules():+ impl = getattr(module, "impl", None)+ runtime = getattr(impl, "runtime", None)+ if isinstance(impl, NVFP4Impl) and runtime is not None:+ if not any(runtime is other for other in seen):+ seen.append(runtime)+ return seen+++class NVFP4WorkerExtension:+ """Named RPC methods for calibration and cache inspection.++ Workers save calibration tensors locally and return only summary metadata.+ """++ def nvfp4_layer_cache_contract(self) -> dict[str, dict[str, int]]:+ """Attest every loaded attention layer through a named safe RPC."""+ layer_cache_dtypes = Counter()+ layer_attention_backends = Counter()+ for module in self.get_model().modules():+ layer_dtype = getattr(module, "kv_cache_dtype", None)+ backend = getattr(module, "attn_backend", None)+ if layer_dtype is None or backend is None:+ continue+ layer_cache_dtypes[str(layer_dtype)] += 1+ layer_attention_backends[str(backend.get_name())] += 1+ return {+ "layer_cache_dtype_counts": dict(layer_cache_dtypes),+ "layer_attention_backend_counts": dict(layer_attention_backends),+ }++ def nvfp4_key_shift_is_active(self) -> bool:+ return all(+ runtime.active_key_shift is not None+ for runtime in _runtimes(self.get_model())+ )++ def nvfp4_set_key_shift_enabled(self, enabled: bool) -> int:+ runtimes = _runtimes(self.get_model())+ for runtime in runtimes:+ runtime.set_key_shift_enabled(enabled)+ return len(runtimes)++ def nvfp4_begin_key_shift_measurement(self) -> int:+ runtimes = _runtimes(self.get_model())+ for runtime in runtimes:+ runtime.begin_key_shift_measurement()+ return len(runtimes)++ def nvfp4_begin_probability_calibration(+ self, tokens: int, multiplier: float, samples: int+ ) -> None:+ from .probability_calibration import ProbabilityCalibration++ if hasattr(self, "_probability_calibration"):+ raise RuntimeError("probability calibration is already active")+ collector = ProbabilityCalibration(tokens, multiplier, samples)+ collector.install(self.get_model())+ self._probability_calibration = collector++ def nvfp4_end_probability_calibration(self, complete: bool = True) -> list[dict] | None:+ collector = self._probability_calibration+ try:+ return collector.finish() if complete else None+ finally:+ collector.close()+ del self._probability_calibration++ def nvfp4_finish_key_shift_measurement(+ self, save_to: str | None = None+ ) -> dict[str, Any] | None:+ runtimes = _runtimes(self.get_model())+ if not runtimes:+ return None+ tokens = runtimes[0].key_shift_tokens+ shift = runtimes[0].finish_key_shift_measurement()+ for other in runtimes[1:]:+ other.finish_key_shift_measurement()++ if save_to is not None:+ from safetensors.torch import save_file++ writer = True+ if torch.distributed.is_initialized():+ from vllm.distributed import get_tp_group++ tp = get_tp_group()+ if tp.world_size > 1:+ total_heads = (+ self.vllm_config.model_config.get_total_num_kv_heads()+ )+ if total_heads != shift.shape[1] * tp.world_size:+ raise ValueError(+ "key-shift export requires unreplicated, evenly "+ "partitioned KV heads"+ )+ counts = tp.all_gather(+ torch.tensor([tokens], dtype=torch.int64, device=shift.device),+ dim=0,+ )+ if not bool((counts == tokens).all()):+ raise ValueError(+ "TP workers disagree on key-shift measurement token counts"+ )+ # vLLM partitions unreplicated KV heads contiguously in+ # TP-rank order. Gather the original-basis means, before+ # the loader applies its rotation and dtype conversion.+ shift = tp.all_gather(shift, dim=1)+ writer = tp.rank_in_group == 0+ if writer:+ save_file(+ {"key_shift": shift.cpu()},+ save_to,+ metadata={"schema": "1", "source": "calibrate_key_shift"},+ )+ return {+ "tokens": int(tokens),+ "shape": list(shift.shape),+ "mean_abs": float(shift.abs().mean()),+ "saved_to": save_to,+ }+++def set_key_shift_enabled(llm, enabled: bool) -> bool:+ """Turn the shift on or off, for measuring one arm against the other.++ The vector survives being turned off, so a single loaded model can answer+ a benchmark both ways. Returns whether the engine had a shift to switch.++ Safe only between requests. Keys already cached were written under+ whichever setting was in force at the time, and a sequence spanning a flip+ would see two constants in one softmax. Calling this between two+ ``llm.generate`` calls satisfies that, since those are synchronous and the+ cache is dropped with the requests that filled it.+ """+ switched = llm.collective_rpc(+ "nvfp4_set_key_shift_enabled", args=(enabled,)+ )+ llm.reset_prefix_cache()+ return any(switched)+++def _long_prompt(llm, texts: list[str], context_tokens: int) -> list[int]:+ """One prompt of about ``context_tokens`` tokens, tiled from ``texts``.++ Tiling repeats the sample rather than extending it, which is fine for a+ mean: the point of the long prompt is to visit high positions, and every+ position gets visited once whatever sits there.+ """+ if not any(text.strip() for text in texts):+ raise ValueError(+ "the model returned no calibration text to tile, so the mean "+ "would be measured over separators"+ )+ unit = llm.get_tokenizer().encode(+ "\n\n".join(texts) + "\n\n", add_special_tokens=False+ )+ repeats = -(-context_tokens // len(unit))+ return (unit * repeats)[:context_tokens]+++def calibrate_key_shift(+ llm,+ *,+ seeds: tuple[str, ...] = SEEDS,+ tokens_per_seed: int = TOKENS_PER_SEED,+ context_tokens: int = CONTEXT_TOKENS,+ save_to: str | None = None,+ probability_multiplier: float | None = None,+ probability_samples: int = 8,+) -> dict[str, Any] | None:+ """Calibrate static key smoothing from a long prefill.++ Returns what was measured -- token count, shape, mean magnitude -- or+ ``None`` if the engine is not running the NVFP4 cache and there was+ nothing to calibrate.++ ``save_to`` writes a reusable sidecar. Pass its path as+ ``additional_config["nvfp4_key_smoothing"]`` when building another engine+ for the same checkpoint. Unreplicated TP head slices are gathered in rank+ order and only TP rank zero writes the global sidecar. Export requires one+ TP group, without pipeline/data parallelism. Without saving, each worker+ keeps its own mean.++ With ``probability_multiplier`` set, also fit the probability threshold+ tau from BF16 Q/K samples in the same long prefill. This requires an eager+ calibration engine so attention hooks execute. The returned+ ``probability_scaling`` report uses leave-one-query-position-out checks.+ """+ from vllm import SamplingParams, TokensPrompt++ if probability_multiplier is not None and not llm.llm_engine.vllm_config.model_config.enforce_eager:+ raise ValueError("probability calibration requires an eager engine")++ try:+ found = llm.collective_rpc("nvfp4_begin_key_shift_measurement")+ except AttributeError as error:+ raise RuntimeError(+ "the engine was built without "+ f'worker_extension_cls="{WORKER_EXTENSION_CLS}", so the K mean '+ "cannot be read out of the worker"+ ) from error+ if not any(found):+ return None+ if save_to is not None:+ config = llm.llm_engine.vllm_config+ parallel = config.parallel_config+ tp_size = parallel.tensor_parallel_size+ heads = config.model_config.get_total_num_kv_heads()+ if (+ not all(found)+ or len(found) != tp_size+ or parallel.pipeline_parallel_size != 1+ or parallel.data_parallel_size != 1+ or heads < tp_size+ or heads % tp_size+ ):+ raise ValueError(+ "key-shift export requires one TP group with unreplicated, "+ "evenly partitioned KV heads and no pipeline/data parallelism"+ )++ sampled = llm.generate(+ list(seeds),+ SamplingParams(+ temperature=0.0, top_p=1.0, max_tokens=tokens_per_seed, seed=42+ ),+ use_tqdm=False,+ )++ budget = min(+ context_tokens, llm.llm_engine.model_config.max_model_len - 1+ )+ if probability_multiplier is not None:+ llm.collective_rpc("nvfp4_begin_probability_calibration",+ args=(budget, probability_multiplier, probability_samples))+ try:+ llm.generate(+ TokensPrompt(+ prompt_token_ids=_long_prompt(+ llm,+ [output.prompt + output.outputs[0].text for output in sampled],+ budget,+ )+ ),+ SamplingParams(temperature=0.0, max_tokens=1),+ use_tqdm=False,+ )+ except BaseException:+ if probability_multiplier is not None:+ llm.collective_rpc("nvfp4_end_probability_calibration", args=(False,))+ raise+ probability_report = None+ if probability_multiplier is not None:+ from .probability_calibration import fit_probability_threshold++ samples = llm.collective_rpc("nvfp4_end_probability_calibration")+ if len(samples) != len(found) or any(+ [row["position"] for row in worker] != [row["position"] for row in samples[0]]+ for worker in samples+ ):+ raise ValueError("TP workers disagree on probability calibration samples")+ probability_report = fit_probability_threshold(samples, probability_multiplier)++ measurements = llm.collective_rpc(+ "nvfp4_finish_key_shift_measurement", args=(save_to,)+ )+ measured = measurements[0]+ if save_to is not None and len(found) > 1:+ if len(measurements) != len(found) or any(+ item is None+ or item["shape"] != measured["shape"]+ or item["tokens"] != measured["tokens"]+ or item["saved_to"] != save_to+ for item in measurements+ ):+ raise ValueError("TP workers disagree on the exported key-shift measurement")+ if probability_report is not None:+ measured = {**measured, "probability_scaling": probability_report}++ # Every key cached during the measurement was stored unshifted, and the+ # next request must not read one. With prefix caching refused -- which the+ # NVFP4 engine configuration requires anyway -- a finished request leaves+ # nothing behind, but saying so costs nothing and does not depend on that.+ llm.reset_prefix_cache()+ return measureddiff --git a/src/nvfp4_vllm/control.py b/src/nvfp4_vllm/control.pynew file mode 100644index 0000000..9dd99e8--- /dev/null+++ b/src/nvfp4_vllm/control.py@@ -0,0 +1,561 @@+"""Per-step bookkeeping for the BF16 tail slots, resolved entirely on device.++A sequence keeps its first 16 tokens in a permanent BF16 attention sink. The+FP4 cache keeps vLLM's original 128-token page grid, including a quantized copy+of those tokens in page zero; decode masks that copy so the sink contributes+exactly once. Keeping the old grid matters because FP4 P and online softmax are+page-local and therefore not invariant to moving every page boundary.++A suffix whose length is not a multiple of the page size is a partial page+that cannot be quantized, because V is packed along the token axis a page at a+time. Those trailing tokens stay in a rolling BF16 buffer until the page+fills. The sink and rolling page share one buffer per running request: a+*slot*.++vLLM never learns that these buffers exist, so it never tells us when one is+dead. It cannot: ``CommonAttentionMetadata`` carries no request identity, and+``finished_req_ids`` lives in ``SchedulerOutput``, which no attention backend+sees. Slot lifetime therefore has to be inferred from what does arrive each+step, which is three device tensors.++The inference keys on ``block_table[row, 0]``, the physical block holding+logical page 0. It is fixed for the life of a request and unique among live+requests, whereas the batch row index is not: ``InputBatch.condense()`` slides+the last live row into the hole a finished request leaves.++Slots are reclaimed lazily. A slot is never released because its key is absent+from a step: the scheduler can leave a live request out of a batch when its+token budget runs out, and releasing that request's tail would silently lose+the only copy of its most recent keys and values. Instead a slot is taken only+when a new key needs one, and the victim is the one whose owner has been unseen+longest. The stronger failure mode this replaces is still detected: a row that+has computed tokens but matches no slot is reported through ``error_code``.++A row is live when its sequence length is nonzero and its key is not vLLM's+null block. Neither test is redundant. Padding rows, which vLLM appends to+reach a captured width for full CUDA graphs, get a zero sequence length but+keep whatever block table entry ``condense()`` left behind, so counting rows+would hand a slot to a request that finished several steps ago. The startup+profile run goes the other way: it reports a full batch of long sequences over+an untouched block table, so every row keys on block 0, which the block pool+reserves as its null block and never allocates. Those rows are not requests.++Everything runs in a single CTA. With a slot table this small the whole+matching problem is a ``[rows, slots]`` comparison held in registers, and one+CTA can mutate the table without grid-wide synchronization.+"""++from __future__ import annotations++from dataclasses import dataclass++import torch+import triton+import triton.language as tl+from nvfp4_decode_kernel.interface import SINK_TOKENS+++PAGE_SIZE = 128+TAIL_BUFFER_TOKENS = PAGE_SIZE + SINK_TOKENS+++def split_sequence_length(+ sequence_length: int,+ *,+ sink_tokens: int = SINK_TOKENS,+ page_size: int = PAGE_SIZE,+) -> tuple[int, int, int]:+ """Describe the BF16 sink over the unchanged FP4 page partition.++ The sink is duplicated in page zero once that page is promoted. Decode+ masks the FP4 copy, so unlike disjoint storage classes these three lengths+ need not sum to the logical sequence length.+ """+ if sequence_length < 0:+ raise ValueError(+ f"sequence_length must be nonnegative, got {sequence_length}"+ )+ if sink_tokens < 0:+ raise ValueError(f"sink_tokens must be nonnegative, got {sink_tokens}")+ if page_size < 1:+ raise ValueError(f"page_size must be positive, got {page_size}")++ sink = min(sequence_length, sink_tokens)+ fp4 = (+ ((sequence_length - 1) // page_size) * page_size+ if sequence_length+ else 0+ )+ return sink, fp4, sequence_length - fp4++# Largest supported slot table. BF16 sink/tail storage grows with slot count,+# while the single-CTA control kernel compares a square matrix of rows and+# slots. Larger tables cause excessive register spilling and require a+# different control-kernel layout.+MAX_SUPPORTED_SLOTS = 128++FREE_KEY = -1+# vLLM's block pool takes block 0 out of the free list at startup and hands it+# to no one, so a row keyed on it is not a request. Dummy batches read it+# straight out of the zero-initialized block table.+NULL_BLOCK = 0+# Marks a row or token that owns no tail slot. A real slot index would be+# indistinguishable from a live entry, so the write path would extend some+# other request's tail with tokens that are not its own.+INACTIVE_ROW = -1++# Sticky bit flags in ``error_code``. Nothing on the hot path reads them;+# reading one costs a host synchronization. They exist so that a violated+# assumption is attributable after the fact rather than silent.+ERR_CONTINUATION_PREFILL = 1+ERR_SLOT_LOST = 2+ERR_STALE_SLOT_HISTORY = 4+ERR_NO_FREE_SLOT = 8+ERR_DUPLICATE_KEY = 16+ERR_PROMOTION_COLUMN = 32++ERROR_NAMES = {+ ERR_CONTINUATION_PREFILL: "a prompt arrived split across steps",+ ERR_SLOT_LOST: "a row with history matched no slot",+ ERR_STALE_SLOT_HISTORY: "a matched slot's length does not continue",+ ERR_NO_FREE_SLOT: "a row needing a slot found none free",+ ERR_DUPLICATE_KEY: "two live rows share a physical block",+ ERR_PROMOTION_COLUMN: "a promotion column is past the block table",+}++_TOKEN_BLOCK = 256++# Triton cannot close over plain Python globals, only over constexpr ones.+_FREE = tl.constexpr(FREE_KEY)+_NULL_BLOCK = tl.constexpr(NULL_BLOCK)+_INACTIVE = tl.constexpr(INACTIVE_ROW)+_E_CONTINUATION_PREFILL = tl.constexpr(ERR_CONTINUATION_PREFILL)+_E_SLOT_LOST = tl.constexpr(ERR_SLOT_LOST)+_E_STALE_SLOT_HISTORY = tl.constexpr(ERR_STALE_SLOT_HISTORY)+_E_NO_FREE_SLOT = tl.constexpr(ERR_NO_FREE_SLOT)+_E_DUPLICATE_KEY = tl.constexpr(ERR_DUPLICATE_KEY)+_E_PROMOTION_COLUMN = tl.constexpr(ERR_PROMOTION_COLUMN)+++def _control_kernel(+ block_table_ptr,+ block_table_stride,+ block_table_columns,+ seq_lens_ptr,+ query_start_loc_ptr,+ slot_keys_ptr,+ slot_last_seq_ptr,+ slot_last_seen_ptr,+ step_ptr,+ row_to_slot_ptr,+ token_to_slot_ptr,+ seqused_sink_ptr,+ seqused_fp4_ptr,+ seqused_residual_ptr,+ promotion_source_tokens_ptr,+ promotion_pages_ptr,+ error_code_ptr,+ num_reqs,+ num_actual_tokens,+ NUM_SLOTS: tl.constexpr,+ SINK: tl.constexpr,+ TAIL_TOKENS: tl.constexpr,+ PAGE: tl.constexpr,+ BLOCK: tl.constexpr,+ TOKEN_BLOCK: tl.constexpr,+):+ step = tl.load(step_ptr) + 1+ tl.store(step_ptr, step)++ idx = tl.arange(0, BLOCK)+ in_batch = idx < num_reqs+ is_slot = idx < NUM_SLOTS++ # Column 0 of the block table, which is a strided read: vLLM owns the+ # tensor and reshaping it per step would cost an allocation and a copy.+ key = tl.load(block_table_ptr + idx * block_table_stride, mask=in_batch, other=_FREE)+ seq = tl.load(seq_lens_ptr + idx, mask=in_batch, other=0)+ q_begin = tl.load(query_start_loc_ptr + idx, mask=in_batch, other=0)+ q_end = tl.load(query_start_loc_ptr + idx + 1, mask=in_batch, other=0)+ query_len = q_end - q_begin++ # ``num_reqs`` only bounds how far the input tensors may be read. A row is+ # a request only if vLLM gave it both a length and a block: padding rows+ # keep a stale block table entry under a zeroed length, and the rows of a+ # dummy batch keep a nonzero length over the null block.+ live = in_batch & (seq > 0) & (key != _NULL_BLOCK)+ # Tokens already in the cache before this step. seq_lens is written as+ # num_computed_tokens + num_scheduled_tokens, so it already counts the+ # tokens this step will store.+ computed = seq - query_len++ slot_key = tl.load(slot_keys_ptr + idx, mask=is_slot, other=_FREE)+ slot_last_seq = tl.load(slot_last_seq_ptr + idx, mask=is_slot, other=-1)+ slot_last_seen = tl.load(slot_last_seen_ptr + idx, mask=is_slot, other=-1)++ # A dead row reads FREE_KEY, which is also an unused slot's key, so the+ # liveness of the row has to be part of the match.+ #+ # Reduced with min rather than a sum of matching indices: two live rows+ # carrying the same key is a broken invariant, not an impossible state, and+ # a sum would answer with an index outside the table. Min answers with a+ # real slot, and the duplicate itself is reported below.+ match = (key[:, None] == slot_key[None, :]) & live[:, None] & is_slot[None, :]+ matched_slot = tl.min(tl.where(match, idx[None, :], NUM_SLOTS), axis=1)+ has_match = matched_slot < NUM_SLOTS+ owns_match = match & (idx[None, :] == matched_slot[:, None])+ matched_last_seq = tl.sum(tl.where(owns_match, slot_last_seq[None, :], 0), axis=1)+ claimed = tl.sum(owns_match.to(tl.int32), axis=0) > 0++ # Rows without a slot take one of the slots no row claimed this step,+ # oldest first. Ranking both sides and pairing the ranks avoids a serial+ # scan over the table.+ needs_slot = live & (has_match == 0)+ need_rank = tl.cumsum(needs_slot.to(tl.int32), axis=0) - 1+ free = is_slot & (claimed == 0)+ older = free[None, :] & (+ (slot_last_seen[None, :] < slot_last_seen[:, None])+ | (+ (slot_last_seen[None, :] == slot_last_seen[:, None])+ & (idx[None, :] < idx[:, None])+ )+ )+ free_rank = tl.sum(older.to(tl.int32), axis=1)+ pick = (+ needs_slot[:, None]+ & free[None, :]+ & (free_rank[None, :] == need_rank[:, None])+ )+ allocated = tl.sum(tl.where(pick, idx[None, :], 0), axis=1)+ got_slot = tl.sum(pick.to(tl.int32), axis=1) > 0++ slot = tl.where(has_match, matched_slot, allocated)+ slot = tl.where(live, slot, _INACTIVE)++ errors = 0+ # A prompt split across steps would arrive with history and more than one+ # query token. Guarded against at configuration time; reported here so a+ # regression is attributable.+ errors += tl.where(+ tl.sum((live & (query_len > 1) & (computed > 0)).to(tl.int32), axis=0) > 0,+ _E_CONTINUATION_PREFILL,+ 0,+ )+ # A row that has already computed tokens but matches no slot has lost its+ # tail. This is exactly the failure mode eager reclamation would cause.+ errors += tl.where(+ tl.sum((needs_slot & (computed > 0)).to(tl.int32), axis=0) > 0,+ _E_SLOT_LOST,+ 0,+ )+ # A matched slot whose recorded length does not continue into this step+ # means the block id was recycled, and only a brand new request can be+ # given a recycled block, so it should have no history to continue.+ errors += tl.where(+ tl.sum(+ (has_match & (matched_last_seq != computed) & (computed > 0)).to(tl.int32),+ axis=0,+ )+ > 0,+ _E_STALE_SLOT_HISTORY,+ 0,+ )+ # Unreachable through the caller, which refuses more rows than slots: a+ # needy row always has an unclaimed slot waiting. This checks the rank+ # pairing above rather than the input.+ errors += tl.where(+ tl.sum((needs_slot & (got_slot == 0)).to(tl.int32), axis=0) > 0,+ _E_NO_FREE_SLOT,+ 0,+ )+ # Two live rows sharing a physical block would have to share a tail, which+ # no pair of live requests can legitimately need. Everything above answers+ # in range regardless, but the answer is meaningless.+ same_key = (key[:, None] == key[None, :]) & live[:, None] & live[None, :]+ errors += tl.where(+ tl.max(tl.sum(same_key.to(tl.int32), axis=1), axis=0) > 1,+ _E_DUPLICATE_KEY,+ 0,+ )++ # Slot state follows its owner. Slots nobody claimed keep their old key so+ # a request that sits out a step can still find its tail. Min again, so a+ # slot two rows landed on takes one of them rather than their sum.+ owns = (slot[:, None] == idx[None, :]) & live[:, None] & is_slot[None, :]+ owned = tl.sum(owns.to(tl.int32), axis=0) > 0+ owner_row = tl.min(tl.where(owns, idx[:, None], BLOCK), axis=0)+ sole_owner = owns & (idx[:, None] == owner_row[None, :])+ owner_key = tl.sum(tl.where(sole_owner, key[:, None], 0), axis=0)+ owner_seq = tl.sum(tl.where(sole_owner, seq[:, None], 0), axis=0)+ tl.store(slot_keys_ptr + idx, tl.where(owned, owner_key, slot_key), mask=is_slot)+ tl.store(+ slot_last_seq_ptr + idx,+ tl.where(owned, owner_seq, slot_last_seq),+ mask=is_slot,+ )+ tl.store(+ slot_last_seen_ptr + idx,+ tl.where(owned, step, slot_last_seen),+ mask=is_slot,+ )++ tl.store(row_to_slot_ptr + idx, slot, mask=is_slot)++ # The first SINK tokens stay in the fixed BF16 sink, while FP4 retains the+ # original vLLM page grid. Page zero eventually contains a duplicate copy+ # that decode masks. Not shifting the grid keeps page-local FP4-P rounding+ # and online-softmax order unchanged for the rest of the sequence.+ # Dead rows are zeroed so graph padding attends to nothing.+ sink = tl.where(live, tl.minimum(seq, SINK), 0)+ live_seq = tl.where(live, seq, 0)+ seq_safe = tl.where(live_seq > 0, live_seq, 1)+ fp4 = tl.where(+ live_seq > 0, ((seq_safe - 1) // PAGE) * PAGE, 0+ )+ tl.store(seqused_sink_ptr + idx, sink, mask=is_slot)+ tl.store(seqused_fp4_ptr + idx, fp4, mask=is_slot)+ tl.store(seqused_residual_ptr + idx, live_seq - fp4, mask=is_slot)++ # Where promotion would read and write, were this the step that filled the+ # tail. Derived here because everything it needs is already in registers:+ # one more gather beats a second kernel launch on a path taken every step.+ #+ # ``column`` is the logical page the tail is filling, and a tail that is+ # exactly a page long is that page, complete. The other rows answer -1,+ # which is what tells the quantizer to leave them alone.+ column = fp4 // PAGE+ crossing = live & (seq > 0) & (seq % PAGE == 0)+ # Reaching past the block table would gather the next row's block id and+ # promote a whole page onto some other request's cache. Structurally that+ # cannot happen — vLLM allocated the block holding the row's last token —+ # but it is exactly the kind of damage nothing downstream could attribute.+ in_bounds = column < block_table_columns+ errors += tl.where(+ tl.sum((crossing & (in_bounds == 0)).to(tl.int32), axis=0) > 0,+ _E_PROMOTION_COLUMN,+ 0,+ )+ tl.store(+ promotion_source_tokens_ptr + idx,+ tl.where(live, slot * TAIL_TOKENS, 0),+ mask=is_slot,+ )+ tl.store(+ promotion_pages_ptr + idx,+ tl.load(+ block_table_ptr + idx * block_table_stride + column,+ mask=crossing & in_bounds,+ other=_INACTIVE,+ ),+ mask=is_slot,+ )++ tl.atomic_or(error_code_ptr, errors)++ # Bounded by the tokens actually present, so a steady-state decode step+ # runs one iteration.+ for base in tl.range(0, num_actual_tokens, TOKEN_BLOCK):+ offs = base + tl.arange(0, TOKEN_BLOCK)+ in_range = offs < num_actual_tokens+ belongs = (+ (offs[:, None] >= q_begin[None, :])+ & (offs[:, None] < q_end[None, :])+ & live[None, :]+ )+ tl.store(+ token_to_slot_ptr + offs,+ tl.sum(tl.where(belongs, slot[None, :], 0), axis=1),+ mask=in_range,+ )+++@dataclass(frozen=True)+class ControlOutputs:+ """Views into the control plane's scratch, valid until the next step.++ Nothing here may be read on the host during a decode step; every tensor+ stays on device and is consumed by later kernels.+ """++ row_to_slot: torch.Tensor+ token_to_slot: torch.Tensor+ # Full width so a captured decode sees one stable-shape sink-length tensor.+ seqused_sink: torch.Tensor+ seqused_fp4: torch.Tensor+ seqused_residual: torch.Tensor+ # Full width like seqused_sink: promotion launches over the whole table so+ # that its shape does not follow the batch.+ promotion_source_tokens: torch.Tensor+ promotion_pages: torch.Tensor+ error_code: torch.Tensor+++class ControlPlaneError(RuntimeError):+ """An invariant the slot table reported through ``error_code``."""+++class ControlPlane:+ """Slot table and per-step scratch for one engine.++ All buffers are allocated once at the high-water mark. Reallocating inside+ a step would both cost a synchronization and move the addresses a captured+ graph would have baked in.+ """++ def __init__(+ self,+ *,+ max_num_seqs: int,+ max_num_batched_tokens: int,+ device: torch.device | str,+ page_size: int = PAGE_SIZE,+ ) -> None:+ if max_num_seqs < 1:+ raise ValueError(f"max_num_seqs must be positive, got {max_num_seqs}")+ if max_num_seqs > MAX_SUPPORTED_SLOTS:+ raise ValueError(+ f"max_num_seqs={max_num_seqs} exceeds the validated "+ f"{MAX_SUPPORTED_SLOTS}-slot control-plane ceiling"+ )+ if max_num_batched_tokens < max_num_seqs:+ raise ValueError(+ f"max_num_batched_tokens ({max_num_batched_tokens}) must cover "+ f"one token per row ({max_num_seqs})"+ )++ self.num_slots = max_num_seqs+ self.max_num_batched_tokens = max_num_batched_tokens+ self.page_size = page_size+ self.device = torch.device(device)+ # Triton indexes the table with tl.arange, which needs a power of two.+ self._block = triton.next_power_of_2(max_num_seqs)++ def _slots(dtype: torch.dtype, fill) -> torch.Tensor:+ return torch.full(+ (self.num_slots,), fill, dtype=dtype, device=self.device+ )++ self.slot_keys = _slots(torch.int32, FREE_KEY)+ # -1 is not a reachable computed-token count, so a slot handed out for+ # the first time never looks like a continuation of anything.+ self.slot_last_seq = _slots(torch.int32, -1)+ # int64 so a long-running server cannot wrap the LRU ordering.+ self.slot_last_seen = _slots(torch.int64, -1)+ self.step = torch.zeros(1, dtype=torch.int64, device=self.device)++ self.row_to_slot = _slots(torch.int32, INACTIVE_ROW)+ self.seqused_sink = _slots(torch.int32, 0)+ self.seqused_fp4 = _slots(torch.int32, 0)+ self.seqused_residual = _slots(torch.int32, 0)+ self.promotion_source_tokens = _slots(torch.int32, 0)+ self.promotion_pages = _slots(torch.int32, INACTIVE_ROW)+ self.error_code = torch.zeros(1, dtype=torch.int32, device=self.device)+ self.token_to_slot = torch.full(+ (max_num_batched_tokens,),+ INACTIVE_ROW,+ dtype=torch.int32,+ device=self.device,+ )++ def reset(self) -> None:+ """Forget every slot. Only valid when no request is in flight."""+ self.slot_keys.fill_(FREE_KEY)+ self.slot_last_seq.fill_(-1)+ self.slot_last_seen.fill_(-1)+ self.step.zero_()+ self.error_code.zero_()++ def raise_for_errors(self) -> None:+ """Read the sticky error word and raise if the kernel reported anything.++ Reading it costs a host synchronization, so nothing on the normal path+ calls this — see ``NVFP4_DEBUG``. The flags are sticky for the life of+ the engine, so a caller that reads only at the end still learns+ everything that went wrong, just not when.+ """+ code = int(self.error_code.item())+ if not code:+ return+ named = [+ text for bit, text in ERROR_NAMES.items() if code & bit+ ]+ unknown = code & ~sum(ERROR_NAMES)+ if unknown:+ named.append(f"unrecognized flags {unknown:#x}")+ raise ControlPlaneError(+ f"the slot control plane reported error_code {code}: "+ + "; ".join(named)+ )++ def prepare(+ self,+ *,+ block_table: torch.Tensor,+ seq_lens: torch.Tensor,+ query_start_loc: torch.Tensor,+ num_reqs: int,+ num_actual_tokens: int,+ ) -> ControlOutputs:+ """Advance the slot table by one step and return this step's mapping.++ ``block_table`` is vLLM's ``[rows, max_blocks]`` tensor; only column 0+ is read, as the identity of the row's request. ``seq_lens`` must+ already include the tokens this step will write, as vLLM's model runner+ computes it, and a row whose length is zero is treated as padding.+ ``num_reqs`` and ``num_actual_tokens`` are host values the caller+ already has; nothing is read back from the device.+ """+ if block_table.ndim != 2:+ raise ValueError(f"block_table must be 2-D, got {block_table.ndim}-D")+ if num_reqs > self.num_slots:+ raise ValueError(+ f"{num_reqs} rows exceeds the {self.num_slots} tail slots"+ )+ if num_actual_tokens > self.max_num_batched_tokens:+ raise ValueError(+ f"{num_actual_tokens} tokens exceeds the "+ f"{self.max_num_batched_tokens} the scratch was sized for"+ )++ _control_kernel[(1,)](+ block_table,+ block_table.stride(0),+ block_table.shape[1],+ seq_lens,+ query_start_loc,+ self.slot_keys,+ self.slot_last_seq,+ self.slot_last_seen,+ self.step,+ self.row_to_slot,+ self.token_to_slot,+ self.seqused_sink,+ self.seqused_fp4,+ self.seqused_residual,+ self.promotion_source_tokens,+ self.promotion_pages,+ self.error_code,+ num_reqs,+ num_actual_tokens,+ NUM_SLOTS=self.num_slots,+ SINK=SINK_TOKENS,+ TAIL_TOKENS=TAIL_BUFFER_TOKENS,+ PAGE=self.page_size,+ BLOCK=self._block,+ TOKEN_BLOCK=_TOKEN_BLOCK,+ num_warps=(+ 32 if self._block == 128 else max(4, self._block // 8)+ ),+ )++ return ControlOutputs(+ row_to_slot=self.row_to_slot[:num_reqs],+ token_to_slot=self.token_to_slot[:num_actual_tokens],+ seqused_sink=self.seqused_sink,+ seqused_fp4=self.seqused_fp4[:num_reqs],+ seqused_residual=self.seqused_residual[:num_reqs],+ promotion_source_tokens=self.promotion_source_tokens,+ promotion_pages=self.promotion_pages,+ error_code=self.error_code,+ )diff --git a/src/nvfp4_vllm/engine.py b/src/nvfp4_vllm/engine.pynew file mode 100644index 0000000..0bffbe7--- /dev/null+++ b/src/nvfp4_vllm/engine.py@@ -0,0 +1,95 @@+"""Build a vLLM engine with the paper's static key smoothing."""++from __future__ import annotations++import os+from typing import Any++from .backend import PAGE_SIZE+from .calibrate import WORKER_EXTENSION_CLS, calibrate_key_shift+from .guards import DEFAULT_SLOTS+from .runtime import KEY_SHIFT_ENV+++REQUIRED_ENGINE_KWARGS: dict[str, Any] = {+ "kv_cache_dtype": "nvfp4",+ "attention_config": {"backend": "CUSTOM"},+ "block_size": PAGE_SIZE,+ "enable_prefix_caching": False,+ "enable_chunked_prefill": False,+}+++def build_llm(+ model: str, *, additional_config: dict[str, Any] | None = None, **kwargs: Any+):+ """Build the FP4 backend with options in ``additional_config``.++ ``nvfp4_key_smoothing`` accepts ``"auto"`` (the default), a calibration+ file path, or ``None`` to disable smoothing. Automatic calibration measures+ the static per-head key mean before serving requests.+ """+ additional = dict(additional_config or {})+ smoothing = additional.setdefault("nvfp4_key_smoothing", "auto")+ if smoothing is not None:+ smoothing = os.fsdecode(smoothing)+ additional["nvfp4_key_smoothing"] = smoothing+ # Source checkouts may not have the installed plugin entry point.+ from . import register as register_custom_backend++ register_custom_backend()++ # CuTe may initialize the CUDA driver before PyTorch detects it. Spawn+ # workers so they do not inherit that driver state through fork.+ os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")++ from vllm import LLM++ contradicted = {+ name: kwargs[name]+ for name, required in REQUIRED_ENGINE_KWARGS.items()+ if name in kwargs and kwargs[name] != required+ }+ if contradicted:+ wanted = {name: REQUIRED_ENGINE_KWARGS[name] for name in contradicted}+ raise ValueError(+ f"this backend requires {wanted}, and was given {contradicted}; "+ "an engine holding only some of these settings still starts, and "+ "serves from a different cache"+ )++ engine = {**REQUIRED_ENGINE_KWARGS, **kwargs, "model": model,+ "additional_config": additional}+ # Reserve one BF16 tail slot per concurrent request.+ engine.setdefault("max_num_seqs", DEFAULT_SLOTS)+ # With chunking disabled, the entire prompt must fit in one batch.+ if engine.get("max_model_len") is not None:+ engine.setdefault(+ "max_num_batched_tokens",+ max(engine["max_model_len"], engine["max_num_seqs"]),+ )++ # Workers read the calibration path during engine construction.+ if smoothing is None or smoothing == "auto":+ os.environ.pop(KEY_SHIFT_ENV, None)+ else:+ os.environ[KEY_SHIFT_ENV] = smoothing++ supplied = engine.get("worker_extension_cls")+ if supplied is None:+ # Also provides the smoothing toggle for saved calibrations.+ engine["worker_extension_cls"] = WORKER_EXTENSION_CLS+ elif smoothing == "auto" and supplied != WORKER_EXTENSION_CLS:+ raise ValueError(+ f"calibrating key smoothing needs worker_extension_cls="+ f"{WORKER_EXTENSION_CLS!r}, and vLLM accepts one; either pass a "+ "sidecar path instead or mix that class into your own"+ )++ llm = LLM(**engine)+ if smoothing == "auto" and calibrate_key_shift(llm) is None:+ raise RuntimeError(+ "the engine came up without the NVFP4 cache, so there was no K "+ "mean to measure"+ )+ return llmdiff --git a/src/nvfp4_vllm/guards.py b/src/nvfp4_vllm/guards.pynew file mode 100644index 0000000..7866fd0--- /dev/null+++ b/src/nvfp4_vllm/guards.py@@ -0,0 +1,305 @@+"""Engine configuration the NVFP4 KV cache path refuses to run under.++These checks cannot live in ``AttentionBackend.validate_configuration``:+``_cached_get_attn_backend`` memoizes on ``AttentionSelectorConfig``, which+carries none of the fields below, so a second engine in the same process would+reuse the first one's verdict and never be validated. ``NVFP4Impl.__init__``+runs once per layer per engine and is not cached.+"""++from __future__ import annotations++from typing import NamedTuple++from nvfp4_decode_kernel.scale_mode import DEFAULT_SCALE_MODE, parse_scale_mode_spec, scale_format, scale_rule+from vllm.config import VllmConfig+from vllm.v1.attention.backend import AttentionImpl, AttentionType++from .control import MAX_SUPPORTED_SLOTS+++NVFP4 = "nvfp4"++# Which block-scale candidates the quantizers search, read from vLLM's+# ``additional_config`` — the field vLLM documents for out-of-tree backend+# options, and the only place a value like this can come from that is neither+# an environment switch nor a fork. It joins the engine's config hash, so two+# engines that disagree about it cannot share a compiled artifact. Holds a+# mode name, or a per-tensor mapping of them; see ``resolve_scale_mode``.+SCALE_MODE_KEY = "nvfp4_scale_mode"++# Whether K derives its eight stored scales independently per 16 channels or+# repeats one norm-derived scale across the whole 128-channel vector. Both use+# the identical FP4 payload and E4M3 scale slots.+KEY_SCALE_GEOMETRY_KEY = "nvfp4_key_scale_geometry"+KEY_SCALE_GEOMETRIES = ("local", "gaussian_global")++# How many independently scaled NVFP4 terms represent each decode query.+# This is fixed before profiling because the second term owns persistent+# scratch buffers that must be charged before vLLM sizes the KV cache.+QUERY_TERMS_KEY = "nvfp4_query_terms"++# The shared orthogonal basis applied to post-RoPE Q and cache-bound K.+# ``none`` preserves the existing path exactly; ``hadamard`` uses one+# normalized Sylvester matrix for every layer and head.+ROTATION_KEY = "nvfp4_rotation"++# vLLM runs at most ``max_num_seqs`` requests concurrently, and each running+# request needs one BF16 tail slot. The slot table itself is sized from+# ``max_num_seqs``; this is the width v1 is validated at.+DEFAULT_SLOTS = 32+MAX_SLOTS = MAX_SUPPORTED_SLOTS+++class UnsupportedConfigError(ValueError):+ """An engine configuration the NVFP4 path cannot serve correctly."""+++class ScaleModes(NamedTuple):+ """One block-scale mode per quantized tensor."""++ key: str+ value: str+ query: str+++def resolve_scale_mode(vllm_config: VllmConfig) -> ScaleModes:+ """The block-scale modes this engine quantizes K, V and Q with.++ A bare name gives all three the same one, and that is the only shape a+ served engine should use: a candidate set that differs by tensor makes a+ quality result unattributable, because the clipping the extra candidates+ trade for accuracy acts differently on a key than on a query.++ A ``{"key": ..., "value": ..., "query": ...}`` mapping sets them apart,+ which exists to answer *which* tensor a quality difference came from once+ the uniform arms have found one. All three names are required — a partial+ mapping would leave the rest at a default that is nobody's intent, and the+ resulting arm would look like a valid one. The precedent is ``num_splits``:+ a typed argument, one value in production, every branch reachable from a+ public entry point.++ Unset means ``amax6``, which is what every cache this repository has+ written so far holds. An unknown name is refused here rather than at the+ first quantized token, because by then the engine has loaded a model. So+ is a mapping that mixes NVFP4 and MXFP4: Q meets K in one MMA and P meets+ V in another, and the kernel runs both in one format.+ """+ additional = getattr(vllm_config, "additional_config", None) or {}+ if not isinstance(additional, dict):+ return ScaleModes(*(DEFAULT_SCALE_MODE,) * 3)+ requested = additional.get(SCALE_MODE_KEY, DEFAULT_SCALE_MODE)++ if isinstance(requested, dict):+ if set(requested) != set(ScaleModes._fields):+ raise UnsupportedConfigError(+ f"additional_config[{SCALE_MODE_KEY!r}] as a mapping needs "+ f"exactly {sorted(ScaleModes._fields)}, got {sorted(requested)}"+ )+ modes = ScaleModes(**requested)+ else:+ modes = ScaleModes(*(requested,) * 3)++ for tensor, mode in zip(ScaleModes._fields, modes):+ try:+ scale_rule(mode)+ except (ValueError, TypeError) as error:+ raise UnsupportedConfigError(+ f"additional_config[{SCALE_MODE_KEY!r}] for {tensor}: {error}"+ ) from None+ formats = {scale_format(mode) for mode in modes}+ if len(formats) != 1:+ raise UnsupportedConfigError(+ f"additional_config[{SCALE_MODE_KEY!r}] mixes {sorted(formats)}; "+ "key, value and query must share one scale format"+ )+ return modes+++def check_mxfp4_options(+ scale_modes: ScaleModes,+ *,+ key_scale_geometry: str,+) -> None:+ """Refuse the NVFP4-only cache options under an MXFP4 scale mode.++ Gaussian-global K scaling uses E4M3 scales and has no MXFP4 kernel.+ """+ if scale_format(scale_modes.key) != "mxfp4":+ return+ unsupported = {+ KEY_SCALE_GEOMETRY_KEY: key_scale_geometry != "local",+ }+ named = sorted(key for key, enabled in unsupported.items() if enabled)+ if named:+ raise UnsupportedConfigError(+ f"MXFP4 scale modes do not support {named} yet"+ )+++def resolve_key_scale_geometry(vllm_config: VllmConfig) -> str:+ """How the K write path shares its already-stored scale slots."""+ additional = getattr(vllm_config, "additional_config", None) or {}+ if not isinstance(additional, dict):+ return "local"+ requested = additional.get(KEY_SCALE_GEOMETRY_KEY, "local")+ if requested not in KEY_SCALE_GEOMETRIES:+ raise UnsupportedConfigError(+ f"additional_config[{KEY_SCALE_GEOMETRY_KEY!r}] must be one of "+ f"{list(KEY_SCALE_GEOMETRIES)}, got {requested!r}"+ )+ return requested+++def resolve_query_terms(vllm_config: VllmConfig) -> int:+ """How many NVFP4 terms represent each decode query."""+ additional = getattr(vllm_config, "additional_config", None) or {}+ if not isinstance(additional, dict):+ return 1+ requested = additional.get(QUERY_TERMS_KEY, 1)+ if type(requested) is not int or requested not in (1, 2):+ raise UnsupportedConfigError(+ f"additional_config[{QUERY_TERMS_KEY!r}] must be 1 or 2, got "+ f"{requested!r}"+ )+ return requested+++def resolve_rotation(vllm_config: VllmConfig) -> str:+ """Which shared basis post-RoPE Q and cache-bound K use."""+ additional = getattr(vllm_config, "additional_config", None) or {}+ if not isinstance(additional, dict):+ return "none"+ requested = additional.get(ROTATION_KEY, "none")+ if requested not in ("none", "hadamard", "block_hadamard_64"):+ raise UnsupportedConfigError(+ f"additional_config[{ROTATION_KEY!r}] must be 'none', "+ f"'hadamard', or 'block_hadamard_64', got {requested!r}"+ )+ return requested+++def check_supported(vllm_config: VllmConfig) -> None:+ """Raise unless the engine is configured for the NVFP4 KV cache path.++ A no-op unless ``kv_cache_dtype`` is ``nvfp4``. With any other cache dtype+ the backend is a plain FlashAttention pass-through: none of the machinery+ these constraints protect is in play.+ """+ cache_config = vllm_config.cache_config+ if cache_config is None or cache_config.cache_dtype != NVFP4:+ return++ scheduler_config = vllm_config.scheduler_config+ if scheduler_config is not None and scheduler_config.max_num_seqs > MAX_SLOTS:+ raise UnsupportedConfigError(+ f"max_num_seqs={scheduler_config.max_num_seqs} exceeds the "+ f"{MAX_SLOTS} BF16 tail slots the NVFP4 path preallocates. Pass "+ f"max_num_seqs<={MAX_SLOTS}, or use a BF16 KV cache."+ )++ if cache_config.enable_prefix_caching:+ raise UnsupportedConfigError(+ "prefix caching is not supported by the NVFP4 KV cache: a cache "+ "hit resumes a sequence mid-page, and prefill over an FP4 prefix "+ "is not implemented. Pass enable_prefix_caching=False."+ )++ # Everything below keeps one invariant: a step either runs a whole prompt+ # from nothing, or extends a sequence by exactly one token. The control+ # plane reports a violation through ``error_code``, but only after the+ # tokens have gone somewhere wrong, so the configuration is refused first.+ if scheduler_config is not None and scheduler_config.enable_chunked_prefill:+ raise UnsupportedConfigError(+ "chunked prefill is not supported by the NVFP4 KV cache: the "+ "second chunk of a split prompt resumes mid-page, which needs "+ "prefill over an FP4 prefix. Pass enable_chunked_prefill=False; "+ "vLLM then requires max_num_batched_tokens >= max_model_len so "+ "that no prompt has to be split."+ )++ if scheduler_config is not None and scheduler_config.long_prefill_token_threshold:+ raise UnsupportedConfigError(+ "long_prefill_token_threshold="+ f"{scheduler_config.long_prefill_token_threshold} splits long "+ "prompts even with chunked prefill disabled. Leave it at 0."+ )++ if vllm_config.speculative_config is not None:+ raise UnsupportedConfigError(+ "speculative decoding is not supported by the NVFP4 KV cache: it "+ "verifies several draft tokens per step, and the decode kernel "+ "attends one query token per sequence."+ )++ if cache_config.kv_cache_dtype_skip_layers:+ raise UnsupportedConfigError(+ "kv_cache_dtype_skip_layers="+ f"{cache_config.kv_cache_dtype_skip_layers} leaves some layers on "+ "a different cache dtype. That splits the model into KV cache "+ "groups with their own metadata builders, so the slot table would "+ "advance more than once per step, and it makes vLLM zero freshly "+ "allocated blocks. Leave it empty."+ )++ if cache_config.kv_offloading_size is not None:+ raise UnsupportedConfigError(+ "KV offloading is not supported by the NVFP4 KV cache: it copies "+ "pages by address and size, which does not describe the packed "+ "FP4 layout. Leave kv_offloading_size unset."+ )++ parallel_config = vllm_config.parallel_config+ if parallel_config is not None and parallel_config.use_ubatching:+ raise UnsupportedConfigError(+ "microbatching is not supported by the NVFP4 KV cache: each "+ "ubatch builds its own attention metadata, so the slot table "+ "would advance twice per step and the second half of the batch "+ "would extend tails the first half had already extended. Leave "+ "enable_dbo unset and ubatch_size at 0."+ )++ if parallel_config is not None and parallel_config.pipeline_parallel_size > 1:+ raise UnsupportedConfigError(+ f"pipeline_parallel_size={parallel_config.pipeline_parallel_size} "+ "is not supported by the NVFP4 KV cache: page promotion is driven "+ "from the last attention layer, which a pipeline stage does not "+ "own. Pass pipeline_parallel_size=1."+ )+++def check_layer_supported(impl: AttentionImpl) -> None:+ """Raise unless this attention layer is one the decode kernel can serve.++ Separate from ``check_supported`` because these come from the model rather+ than the engine configuration. Each of them is a modifier the decode kernel+ does not implement, so serving the layer would return plausible attention+ computed against the wrong mask or the wrong scores — wrong quietly, which+ is the failure mode worth spending a check on.+ """+ if impl.attn_type != AttentionType.DECODER:+ raise UnsupportedConfigError(+ f"attention type {impl.attn_type} is not supported by the NVFP4 "+ "KV cache, which is a decoder cache."+ )+ if impl.kv_sharing_target_layer_name is not None:+ raise UnsupportedConfigError(+ "cross-layer KV sharing is not supported by the NVFP4 KV cache: "+ "the BF16 tail is indexed by layer, so a layer that reads another "+ "layer's cache would read its own empty tail."+ )+ # The two above each fail for their own reason and say so. These four fail+ # for one reason, so they are reported together rather than one per run.+ unsupported = {+ "sliding window": impl.sliding_window != (-1, -1),+ "logit soft capping": bool(impl.logits_soft_cap),+ "ALiBi": impl.alibi_slopes is not None,+ "attention sinks": impl.sinks is not None,+ }+ named = [name for name, present in unsupported.items() if present]+ if named:+ raise UnsupportedConfigError(+ f"{', '.join(named)} is not supported by the NVFP4 decode kernel, "+ "which computes unmodified scaled dot-product attention over the "+ "whole cache."+ )diff --git a/src/nvfp4_vllm/impl.py b/src/nvfp4_vllm/impl.pynew file mode 100644index 0000000..ad30ac7--- /dev/null+++ b/src/nvfp4_vllm/impl.py@@ -0,0 +1,405 @@+"""Attention implementation for the NVFP4 KV cache."""++from __future__ import annotations++import hashlib+import os++import torch+from vllm.config import get_current_vllm_config+from vllm.forward_context import get_forward_context+from vllm.logger import init_logger+from vllm.model_executor.layers.attention.attention import (+ get_attention_context,+)+from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func+from vllm.v1.attention.backends.flash_attn import FlashAttentionImpl++from nvfp4_decode_kernel import DecodeTailWrite, fp4_decode+from nvfp4_decode_kernel.interface import SINK_TOKENS++from . import promote+from .guards import (+ NVFP4,+ check_layer_supported,+ check_mxfp4_options,+ check_supported,+ resolve_key_scale_geometry,+ resolve_query_terms,+ resolve_rotation,+ resolve_scale_mode,+)+from .runtime import KEY_SHIFT_ENV, LayerRuntime+from .write import reset_new_request_tails, write_kv++logger = init_logger(__name__)+++def _sidecar_sha256(env_name: str) -> str:+ path = os.environ.get(env_name)+ if not path:+ return "none"+ digest = hashlib.sha256()+ with open(path, "rb") as stream:+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):+ digest.update(chunk)+ return digest.hexdigest()+++class NVFP4Impl(FlashAttentionImpl):+ """Attention over the NVFP4 cache, split by what each row is doing.++ vLLM has sorted the one-token rows to the front of the batch (see+ ``builder.decode_split``), which splits the step cleanly in two. The decode+ prefix reads the FP4 pages and the BF16 tail through ``fp4_decode``. The+ prefill suffix reads neither: with chunked prefill and prefix caching both+ refused, a prompt arrives whole, so FlashAttention attends it against the+ K/V of this same forward pass and never touches the block table.++ Either side can be empty, and usually one is.+ """++ def __init__(self, *args, **kwargs) -> None:+ config = get_current_vllm_config()+ # Ahead of super(), so an unsupported engine configuration is reported+ # instead of whatever FlashAttention's own setup happens to fail on.+ check_supported(config)+ super().__init__(*args, **kwargs)++ cache_config = config.cache_config+ self.nvfp4 = (+ cache_config is not None and cache_config.cache_dtype == NVFP4+ )+ self.num_slots = 0+ self.scale_modes = resolve_scale_mode(config)+ self.key_scale_geometry = resolve_key_scale_geometry(config)+ self.query_terms = resolve_query_terms(config)+ self.rotation = resolve_rotation(config)+ check_mxfp4_options(+ self.scale_modes,+ key_scale_geometry=self.key_scale_geometry,+ )+ if self.nvfp4:+ check_layer_supported(self)+ # FlashAttention turns this on for any quantized cache, and the+ # attention layer then hands the impl an FP8 query. Both paths here+ # need BF16: prefill attends against this pass's own BF16 K/V, and+ # decode quantizes to FP4, not FP8.+ self.supports_quant_query_input = False+ self.num_slots = config.scheduler_config.max_num_seqs+ self.runtime: LayerRuntime | None = None+ self.layer_index = 0++ def _bind_runtime(self, device: torch.device) -> None:+ """Create the model's shared state once and hand it around.++ Called from the profile run, where every attention layer is already+ registered but the KV cache does not exist yet. Allocating here is what+ makes vLLM subtract this memory from the cache budget.+ """+ modules = [+ module+ for module in get_forward_context().no_compile_layers.values()+ if isinstance(getattr(module, "impl", None), NVFP4Impl)+ ]+ if not any(module.impl is self for module in modules):+ raise ValueError(+ "an NVFP4 layer is running outside the forward context that "+ "lists it, so the write path cannot be shared with its peers"+ )+ shapes = {+ (+ module.impl.num_heads,+ module.impl.num_kv_heads,+ module.impl.head_size,+ )+ for module in modules+ }+ if len(shapes) != 1:+ raise ValueError(+ f"NVFP4 needs one attention shape across layers, found {shapes}"+ )+ num_heads, num_kv_heads, head_dim = shapes.pop()++ runtime = LayerRuntime(+ num_layers=len(modules),+ num_slots=self.num_slots,+ num_heads=num_heads,+ num_kv_heads=num_kv_heads,+ head_dim=head_dim,+ device=device,+ query_terms=self.query_terms,+ rotation=self.rotation,+ key_scale_mode=self.scale_modes.key,+ value_scale_mode=self.scale_modes.value,+ key_scale_geometry=self.key_scale_geometry,+ )+ # Both of these are read from configuration that is easy to deliver+ # almost correctly: a scale mode arrives inside ``additional_config``,+ # which resolve_scale_mode abandons for the default when it is not a+ # plain dict, and a key shift arrives as an environment variable that+ # has to survive into a worker process. Either can be absent while the+ # engine comes up looking exactly like one that has them, so an arm+ # that meant to change the cache is otherwise indistinguishable from+ # the control. Say once per engine what the cache is actually running.+ shift = runtime.active_key_shift+ # verl deliberately runs vLLM workers at WARN. This is a correctness+ # attestation rather than routine progress, so keep it visible there:+ # without it a centered arm can complete while providing no evidence+ # that the worker received either its scale mode or its sidecar.+ logger.warning(+ "nvfp4 cache attestation: attention_backend=CUSTOM "+ "kv_cache_dtype=nvfp4 key_shift_enabled=%s key_shift_sha256=%s "+ "q_scale_mode=%s k_scale_mode=%s v_scale_mode=%s "+ "k_scale_geometry=%s "+ "query_terms=%d sink_tokens=%d layers=%d key_shift=%s",+ shift is not None,+ _sidecar_sha256(KEY_SHIFT_ENV),+ self.scale_modes.query,+ self.scale_modes.key,+ self.scale_modes.value,+ runtime.key_scale_geometry,+ runtime.query_terms,+ SINK_TOKENS,+ len(modules),+ "none"+ if shift is None+ else f"loaded mean_abs={float(shift.abs().mean()):.6g}",+ )++ # The index only has to be a stable bijection; dict order follows the+ # order the layers were constructed in, which makes it readable too.+ for index, module in enumerate(modules):+ module.impl.runtime = runtime+ module.impl.layer_index = index++ def do_kv_cache_update(+ self,+ layer: torch.nn.Module,+ key: torch.Tensor,+ value: torch.Tensor,+ kv_cache: torch.Tensor,+ slot_mapping: torch.Tensor,+ ) -> None:+ if not self.nvfp4:+ super().do_kv_cache_update(+ layer, key, value, kv_cache, slot_mapping+ )+ return++ metadata, _, _, _ = get_attention_context(layer.layer_name)+ # Both mean the profile run, where there is no cache to write into:+ # the builder returns no metadata, and the runtime is not bound until+ # the first forward. On a real step neither can be None.+ if metadata is None or self.runtime is None:+ return++ # The first layer speaks for the whole model here: a slot's previous+ # tenant has to be erased from every layer before any of them writes+ # this step, and layers run in the order they were indexed.+ if self.layer_index == 0:+ reset_new_request_tails(+ tail_key=self.runtime.tail_key,+ tail_value=self.runtime.tail_value,+ query_start_loc=metadata.query_start_loc,+ seq_lens=metadata.seq_lens,+ row_to_slot=metadata.row_to_slot,+ )++ if self.runtime.key_shift_sum is not None:+ self.runtime.key_shift_sum[self.layer_index] += (+ key[: metadata.num_actual_tokens].double().sum(dim=0)+ )+ # Counted once for the step, by the same layer that speaks for the+ # model above, because every layer adds the same tokens.+ if self.layer_index == 0:+ self.runtime.key_shift_tokens += metadata.num_actual_tokens++ # Out of place, so prefill still attends this pass's original K. Only+ # what the cache holds is centered and/or rotated, by the writers that+ # all three cache write paths go through.+ runtime = self.runtime+ key, _, _ = runtime.key_write(self.layer_index, key)+ writer, regions = runtime.writer_for(self.layer_index, kv_cache)+ decode_rows = metadata.decode_prefix_rows+ write_kv(+ writer=writer,+ regions=regions,+ key=key,+ value=value,+ source_tokens=metadata.source_tokens,+ destination_pages=metadata.destination_pages,+ query_start_loc=metadata.query_start_loc,+ seq_lens=metadata.seq_lens,+ seqused_fp4=metadata.seqused_fp4,+ row_to_slot=metadata.row_to_slot,+ layer_index=self.layer_index,+ decode_rows=decode_rows,+ first_row=decode_rows if runtime.decode_writes_tail else 0,+ )++ def forward(+ self,+ layer: torch.nn.Module,+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ kv_cache: torch.Tensor,+ attn_metadata,+ output: torch.Tensor | None = None,+ output_scale: torch.Tensor | None = None,+ output_block_scale: torch.Tensor | None = None,+ ) -> torch.Tensor:+ if not self.nvfp4:+ return super().forward(+ layer,+ query,+ key,+ value,+ kv_cache,+ attn_metadata,+ output,+ output_scale,+ output_block_scale,+ )+ if output_scale is not None or output_block_scale is not None:+ raise NotImplementedError(+ "fused output quantization is not supported by the NVFP4 "+ "attention path"+ )+ if self.runtime is None:+ self._bind_runtime(query.device)+ if attn_metadata is None:+ # Profile run. The cache is a placeholder with no storage, so there+ # is nothing to read; zero rather than leave the buffer untouched,+ # since NaN in the residual stream propagates into every later+ # layer's K/V. Counted out of the order below too, which is about+ # steps that write.+ return output.zero_()++ self.runtime.expect_layer(self.layer_index)+ rows = attn_metadata.decode_prefix_rows+ tokens = attn_metadata.decode_prefix_tokens+ if rows:+ self._decode(+ rows, query, key, value, kv_cache, attn_metadata, output+ )+ if tokens < attn_metadata.num_actual_tokens:+ self._prefill(tokens, query, key, value, attn_metadata, output)+ # Last layer, so every layer has now both read the tail as this step's+ # metadata describes it and written its share of whatever filled it.+ if self.layer_index == self.runtime.num_layers - 1:+ promote.launch(attn_metadata, self.runtime)+ return output+++ def _decode(+ self,+ rows: int,+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ kv_cache: torch.Tensor,+ attn_metadata,+ output: torch.Tensor,+ ) -> None:+ """Attention for the one-token rows, over the FP4 pages and the tail.++ Every per-row argument is a prefix slice of this step's arrays, which+ is what the batch reordering buys: no compaction, no scatter, and an+ output that lands where vLLM already wants it.+ """+ key_pages_fp4, key_scales, value_pages_fp4, value_scales = (+ self.runtime.views(self.layer_index, kv_cache)+ )+ pages = attn_metadata.decode_page_columns+ query = self.runtime.transform_query(query[:rows])+ raw_key, rotate_key, key_shift = self.runtime.key_write(+ self.layer_index, key+ )+ tail_write = None+ if self.runtime.decode_writes_tail:+ tail_write = DecodeTailWrite(+ key=raw_key[:rows],+ value=value[:rows],+ tail_key=self.runtime.tail_key[self.layer_index],+ tail_value=self.runtime.tail_value[self.layer_index],+ row_to_slot=attn_metadata.row_to_slot[:rows],+ seq_lens=attn_metadata.seq_lens[:rows],+ seqused_fp4=attn_metadata.seqused_fp4[:rows],+ rotate_key=rotate_key,+ key_shift=key_shift,+ sink_tokens=SINK_TOKENS,+ )+ fp4_decode(+ query=query,+ key_pages_fp4=key_pages_fp4,+ key_scales=key_scales,+ value_pages_fp4=value_pages_fp4,+ value_scales=value_scales,+ fp4_page_table=attn_metadata.block_table[:rows, :pages],+ seqused_fp4=attn_metadata.seqused_fp4[:rows],+ seqused_sink=(+ attn_metadata.seqused_sink[:rows] if SINK_TOKENS else None+ ),+ residual_key_pages_bf16=self.runtime.tail_key[self.layer_index],+ residual_value_pages_bf16=self.runtime.tail_value[+ self.layer_index+ ],+ residual_page_ids=attn_metadata.row_to_slot[:rows],+ seqused_residual=attn_metadata.seqused_residual[:rows],+ softmax_scale=self.scale,+ # The control plane produced these on the device this same step and+ # nothing since could have invalidated them. Checking would mean+ # reading them back, which costs a synchronization per layer.+ trusted_metadata=True,+ query_padded_scratch=self.runtime.query_padded,+ query_fp4_scratch=self.runtime.query_fp4,+ query_scales_scratch=self.runtime.query_scales,+ query_fp4_second_scratch=self.runtime.query_fp4_second,+ query_scales_second_scratch=self.runtime.query_scales_second,+ num_splits=attn_metadata.decode_num_splits,+ split_partial_scratch=self.runtime.split_partial,+ split_lse_scratch=self.runtime.split_lse,+ query_terms=self.runtime.query_terms,+ scale_mode=self.scale_modes.query,+ query_rotation=self.runtime.query_rotation,+ out=output[:rows],+ tail_write=tail_write,+ )++ def _prefill(+ self,+ tokens: int,+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_metadata,+ output: torch.Tensor,+ ) -> None:+ """Attention for the rows that arrived with their whole prompt.++ Cache-free: ``k`` and ``v`` are this pass's own tensors and+ ``cu_seqlens_k`` is ``cu_seqlens_q``, so the FP4 pages this prompt is+ about to be written into are never read. That is what lets prefill run+ against a cache FlashAttention cannot decode.++ ``max_seqlen_q`` is the whole batch's maximum rather than the suffix's.+ The prefills are the long rows, so the two agree unless the batch is+ pure decode, in which case this does not run.+ """+ starts = attn_metadata.prefill_query_start_loc+ num_actual_tokens = attn_metadata.num_actual_tokens+ flash_attn_varlen_func(+ q=query[tokens:num_actual_tokens],+ k=key[tokens:num_actual_tokens],+ v=value[tokens:num_actual_tokens],+ out=output[tokens:num_actual_tokens],+ cu_seqlens_q=starts,+ max_seqlen_q=attn_metadata.max_query_len,+ cu_seqlens_k=starts,+ max_seqlen_k=attn_metadata.max_query_len,+ softmax_scale=self.scale,+ causal=attn_metadata.causal,+ fa_version=self.vllm_flash_attn_version,+ )diff --git a/src/nvfp4_vllm/layout.py b/src/nvfp4_vllm/layout.pynew file mode 100644index 0000000..065b424--- /dev/null+++ b/src/nvfp4_vllm/layout.py@@ -0,0 +1,130 @@+"""Interpretation of a vLLM KV cache block as an NVFP4 page.++vLLM hands out a flat byte page per block and never looks inside it. The four+things that have to live there — packed K, K scales, packed V, V scales — are+laid out one after another *within* each block rather than as four contiguous+region arrays spanning all blocks. Block-major costs each region a page stride,+which the quantizer and the decode kernel both accept, and in exchange a block+stays the self-contained byte range that vLLM believes it is. Anything that+copies a block by address, which today the guardrails forbid but tomorrow may+not, then keeps working.++An MXFP4 page has the same packed regions and half the scale bytes. Its scale+regions keep the NVFP4 size, the tail unused, so a block is the same size in+either format and an engine sizes its cache identically for both.+"""++from __future__ import annotations++import functools++import torch++from nvfp4_decode_kernel.quantize_kv_kernel import (+ quantize_key_pages,+ quantize_value_pages,+)+from nvfp4_decode_kernel.scale_mode import DEFAULT_SCALE_MODE+++PAGE_SIZE = 128+++def _probe_regions(+ num_kv_heads: int, head_dim: int, device: str, scale_mode: str+) -> tuple[torch.Tensor, ...]:+ probe = torch.zeros(+ 2,+ PAGE_SIZE,+ num_kv_heads,+ head_dim,+ dtype=torch.bfloat16,+ device=device,+ )+ return (+ *quantize_key_pages(probe, scale_mode=scale_mode),+ *quantize_value_pages(probe, scale_mode=scale_mode),+ )++[email protected]_cache(maxsize=None)+def region_specs(+ num_kv_heads: int,+ head_dim: int,+ device: str,+ scale_mode: str = DEFAULT_SCALE_MODE,+) -> tuple[tuple, ...]:+ """Ask the quantizers what one page of each region actually looks like.++ The scale factors use a swizzled layout, so their shape and strides are not+ something to rederive here; running the real quantizer on a throwaway page+ is the only way to stay correct if that layout ever changes. A region's+ size is always the NVFP4 one, which an MXFP4 region fits inside.+ """+ regions = _probe_regions(num_kv_heads, head_dim, device, scale_mode)+ sizes = [+ region.stride()[0] * region.element_size()+ for region in _probe_regions(+ num_kv_heads, head_dim, device, DEFAULT_SCALE_MODE+ )+ ]+ return tuple(+ (+ tuple(region.shape[1:]),+ tuple(region.stride()[1:]),+ region.dtype,+ size,+ )+ for region, size in zip(regions, sizes)+ )+++def page_bytes(num_kv_heads: int, head_dim: int, device: str) -> int:+ """Total bytes one block must provide for all four regions."""+ return sum(spec[3] for spec in region_specs(num_kv_heads, head_dim, device))+++def carve(+ kv_cache: torch.Tensor,+ num_kv_heads: int,+ head_dim: int,+ scale_mode: str = DEFAULT_SCALE_MODE,+) -> tuple[torch.Tensor, ...]:+ """View one layer's block array as packed K, K scales, packed V, V scales.++ Each returned view keeps the quantizer's exact layout inside a page and+ replaces only its page stride with the block pitch, which is what both the+ quantizer's indexed writes and the decode kernel's page strides expect.+ """+ if kv_cache.ndim != 2 or kv_cache.element_size() != 1:+ raise ValueError(+ f"expected a [blocks, bytes] byte cache, got {tuple(kv_cache.shape)}"+ f" of {kv_cache.dtype}"+ )+ blocks, pitch = kv_cache.shape+ if kv_cache.stride() != (pitch, 1):+ raise ValueError("the block array must be contiguous")++ specs = region_specs(num_kv_heads, head_dim, str(kv_cache.device), scale_mode)+ required = sum(spec[3] for spec in specs)+ if pitch != required:+ raise ValueError(+ f"a block of {pitch} bytes cannot hold {required} bytes of NVFP4 "+ "page; the backend's page size and this layout disagree"+ )++ flat = kv_cache.view(torch.uint8)+ views, offset = [], kv_cache.storage_offset()+ for shape, strides, dtype, region_bytes in specs:+ if dtype.itemsize != 1:+ # Offsets and strides below are byte counts taken from the probe,+ # which as_strided would read as element counts for a wider dtype.+ raise ValueError(f"region dtype {dtype} is not byte-sized")+ typed = flat if dtype is torch.uint8 else flat.view(dtype)+ views.append(+ typed.as_strided(+ (blocks,) + shape, (pitch,) + strides, offset+ )+ )+ offset += region_bytes+ return tuple(views)diff --git a/src/nvfp4_vllm/metadata.py b/src/nvfp4_vllm/metadata.pynew file mode 100644index 0000000..95ac741--- /dev/null+++ b/src/nvfp4_vllm/metadata.py@@ -0,0 +1,149 @@+"""Per-step attention metadata for the NVFP4 KV cache.++Carries FlashAttention's own metadata unchanged and appends what the control+plane resolved for this step. Subclassing rather than wrapping keeps every+non-NVFP4 layer working: until the decode kernel is wired in, the layers still+run FlashAttention, and they read these objects through the base class's+fields.++Most of the fields below are device tensors written by the control kernel.+Reading any of those on the host costs a synchronization, so nothing outside a+kernel should. The plain ints are the exception, and are marked as such: they+come from CPU-side batch descriptions, never from the device.+"""++from __future__ import annotations++from dataclasses import dataclass, fields++import torch+from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata++from .control import ControlOutputs+++# kw_only because the base class ends in defaulted fields, which would+# otherwise forbid adding required ones.+@dataclass(kw_only=True)+class NVFP4Metadata(FlashAttentionMetadata):+ """FlashAttention metadata plus this step's tail-slot assignment."""++ row_to_slot: torch.Tensor+ """``[num_reqs]`` int32. Tail slot owning each batch row, -1 if the row is+ padding."""++ token_to_slot: torch.Tensor+ """``[num_actual_tokens]`` int32. Tail slot each token belongs to, so the+ KV write can find its destination without a per-row loop."""++ seqused_sink: torch.Tensor+ """``[max_num_seqs]`` int32. Tokens of each sequence held permanently in+ the BF16 sink. Full width and zero for graph-padding rows."""++ seqused_fp4: torch.Tensor+ """``[num_reqs]`` int32. Prefix covered by whole FP4 pages on vLLM's+ original page grid. Page zero includes a masked copy of the BF16 sink."""++ seqused_residual: torch.Tensor+ """``[num_reqs]`` int32. Tokens of each sequence that live in the BF16+ rolling tail. Together with ``seqused_fp4`` this sums to the sequence+ length; ``seqused_sink`` is a duplicated precision overlay."""++ promotion_source_tokens: torch.Tensor+ """``[max_num_seqs]`` int32. First token of each row's tail, as an index+ into one layer's tail buffer."""++ promotion_pages: torch.Tensor+ """``[max_num_seqs]`` int32. Block that a row's tail must be quantized into+ before the next step, -1 for a row whose tail did not fill a page.++ Full width rather than ``num_reqs`` wide, unlike the fields above: this+ pair is the promotion launch's grid, and a grid that followed the batch+ would put a host branch back on a path that has none."""++ decode_prefix_rows: int+ """How many rows of this batch emit exactly one token. vLLM has already+ moved them to the front, so they are rows ``[0, decode_prefix_rows)``. A+ host int, not a device tensor: it comes from ``query_start_loc_cpu``, so+ reading it is free.++ Not the base class's ``num_decode_reqs``, which counts something else+ (rows that carry decode context under decode context parallelism) and is+ left at zero outside that feature."""++ decode_prefix_tokens: int+ """Where the prefill tokens begin. Equal to ``decode_prefix_rows`` while+ every decode row emits one token, but derived rather than assumed."""++ prefill_query_start_loc: torch.Tensor+ """``[num_prefills + 1]`` int32. ``query_start_loc`` rebased on the first+ prefill token, which is what FlashAttention needs once the decode prefix+ has been sliced off. Built here rather than per layer because every layer+ of the step would otherwise recompute the same subtraction on the+ device."""++ decode_page_columns: int+ """How many page-table columns the decode kernel has to walk. The kernel+ reads the table's width as the longest row it could see, which is the+ length it validates ``seqused_fp4`` against and the extent its tile+ scheduler is sized for, so a table left at the model's capacity makes a+ short batch look long.++ The batch's longest row, not the decodes' own: the per-row lengths live on+ the device, and vLLM's host-side copy is materialized by a transfer that+ would cost the step a synchronization. A prefill sharing the step can only+ widen this, never narrow it wrongly. The final page on the original vLLM+ grid is always the rolling tail rather than an FP4 page."""++ decode_num_splits: int+ """How many key partitions the decode kernel splits into, from+ ``builder.decode_num_splits``. A host int chosen from the decode row count+ alone, so a graph captured at a width keeps the count right for every+ batch it later replays."""++ source_tokens: torch.Tensor+ """``[work]`` int32. First token of each full page this step must quantize,+ as an index into the step's flattened K/V."""++ destination_pages: torch.Tensor+ """``[work]`` int32. Block each of those pages belongs in, -1 where the+ grid slot has no work. Paired with ``source_tokens`` and shared by every+ layer, since which pages exist does not depend on the layer."""++ @classmethod+ def from_flash(+ cls,+ base: FlashAttentionMetadata,+ outputs: ControlOutputs,+ *,+ source_tokens: torch.Tensor,+ destination_pages: torch.Tensor,+ decode_prefix_rows: int,+ decode_prefix_tokens: int,+ prefill_query_start_loc: torch.Tensor,+ decode_page_columns: int,+ decode_num_splits: int,+ ) -> "NVFP4Metadata":+ """Carry every FlashAttention field across and append the slot fields.++ Copied field by field rather than mutated in place: the base object is+ what FlashAttention's own builder returned, and a metadata object is+ read by every layer in the step.+ """+ return cls(+ **{field.name: getattr(base, field.name) for field in fields(base)},+ row_to_slot=outputs.row_to_slot,+ token_to_slot=outputs.token_to_slot,+ seqused_sink=outputs.seqused_sink,+ seqused_fp4=outputs.seqused_fp4,+ seqused_residual=outputs.seqused_residual,+ promotion_source_tokens=outputs.promotion_source_tokens,+ promotion_pages=outputs.promotion_pages,+ decode_prefix_rows=decode_prefix_rows,+ decode_prefix_tokens=decode_prefix_tokens,+ prefill_query_start_loc=prefill_query_start_loc,+ decode_page_columns=decode_page_columns,+ decode_num_splits=decode_num_splits,+ source_tokens=source_tokens,+ destination_pages=destination_pages,+ )diff --git a/src/nvfp4_vllm/probability_calibration.py b/src/nvfp4_vllm/probability_calibration.pynew file mode 100644index 0000000..1a4266a--- /dev/null+++ b/src/nvfp4_vllm/probability_calibration.py@@ -0,0 +1,160 @@+"""Fit the probability-scale threshold from bounded calibration Q/K samples."""++from __future__ import annotations++import math+import struct+++def _e4m3_value(code: int) -> float:+ exponent, mantissa = code >> 3, code & 7+ return mantissa / 512 if exponent == 0 else (1 + mantissa / 8) * 2 ** (exponent - 7)+++def threshold_for_code(code: int) -> float:+ """Largest FP32 cut matching E4M3 round-to-nearest-even codes <= code."""+ if code == -1:+ return 0.0+ if not 0 <= code < 126:+ raise ValueError("threshold code must be in [-1, 125]")+ midpoint = (_e4m3_value(code) + _e4m3_value(code + 1)) / 2+ if code % 2:+ bits = struct.unpack("!I", struct.pack("!f", midpoint))[0]+ return struct.unpack("!f", struct.pack("!I", bits - 1))[0]+ return midpoint+++def _combine(samples: list[dict]) -> dict:+ total = {}+ for sample in samples:+ for code, bucket in sample["codes"].items():+ target = total.setdefault(int(code), {"groups": 0, "2": 0.0, "6": 0.0})+ target["groups"] += bucket["groups"]+ for divisor in ("2", "6"):+ target[divisor] += bucket["error_sum"][divisor]+ return total+++def _error(buckets: dict, cut: int) -> float:+ return math.fsum(bucket["2" if code <= cut else "6"] for code, bucket in buckets.items())+++def _best_cut(buckets: dict) -> int:+ return min(range(-1, max(buckets) + 1), key=lambda code: (_error(buckets, code), code))+++def fit_probability_threshold(worker_samples: list[list[dict]], multiplier: float) -> dict:+ """Fit tau on all samples and report leave-one-query-position-out error."""+ if not math.isfinite(multiplier) or multiplier <= 0:+ raise ValueError("probability multiplier must be positive and finite")+ samples = {}+ for worker in worker_samples:+ for sample in worker:+ samples.setdefault(sample["position"], []).append(sample)+ if len(samples) < 2:+ raise ValueError("tau calibration needs at least two query positions")+ grouped = {position: _combine(values) for position, values in samples.items()}+ for buckets in grouped.values():+ if not buckets or any(+ not 0 <= code < 126 or bucket["groups"] < 1+ or any(not math.isfinite(bucket[d]) or bucket[d] < 0 for d in ("2", "6"))+ for code, bucket in buckets.items()+ ):+ raise ValueError("invalid probability calibration statistics")+ total = _combine([sample for worker in worker_samples for sample in worker])+ cut = _best_cut(total)+ folds = []+ for position, held in sorted(grouped.items()):+ training = {+ code: {key: value - held.get(code, {}).get(key, 0)+ for key, value in bucket.items()}+ for code, bucket in total.items()+ }+ training_cut = _best_cut(training)+ folds.append({"position": position, "tau": threshold_for_code(training_cut),+ "squared_error": _error(held, training_cut) / multiplier**2})+ return {+ "tau": threshold_for_code(cut), "c": multiplier,+ "rule": "max(p) <= tau / c", "threshold_code": cut,+ "samples": len(samples), "groups": sum(bucket["groups"] for bucket in total.values()),+ "squared_error": _error(total, cut) / multiplier**2,+ "amax6_squared_error": math.fsum(bucket["6"] for bucket in total.values()) / multiplier**2,+ "validation": "leave_one_query_position_out", "folds": folds,+ "source": "BF16 Q/K, reverse-page online softmax, 16-value probability groups",+ }+++class ProbabilityCalibration:+ """Observe one unchunked prefill and retain only per-code error sums."""++ def __init__(self, tokens: int, multiplier: float, samples: int = 8):+ if tokens < 256 or not 2 <= samples <= tokens - 128:+ raise ValueError("tau calibration needs at least 256 tokens and 2..tokens-128 samples")+ if not math.isfinite(multiplier) or not 0 < multiplier <= 448:+ raise ValueError("probability multiplier must be finite and between 0 and 448")+ self.tokens = tokens+ self.multiplier = multiplier+ self.positions = [128 + i * (tokens - 129) // (samples - 1) for i in range(samples)]+ self.samples = {position: {"position": position, "codes": {}} for position in self.positions}+ self.handles = []+ self.seen = set()+ self.expected_layers = 0++ def install(self, model) -> None:+ from vllm.model_executor.layers.attention.attention import Attention++ try:+ for module in model.modules():+ if isinstance(module, Attention):+ self.handles.append(module.register_forward_pre_hook(self.observe, with_kwargs=True))+ self.expected_layers = len(self.handles)+ if not self.expected_layers:+ raise ValueError("no attention layers found for probability calibration")+ except BaseException:+ self.close()+ raise++ def observe(self, module, args, kwargs) -> None:+ import torch+ from nvfp4_decode_kernel.reference import online_probability_scale_statistics++ if id(module) in self.seen:+ raise RuntimeError("tau calibration requires one complete prefill per attention layer")+ query = args[0] if args else kwargs["query"]+ key = args[1] if len(args) > 1 else kwargs["key"]+ if query.shape[0] < self.tokens or key.shape[0] < self.tokens:+ raise ValueError("tau calibration requires an unchunked prefill")+ heads, kv_heads, dim = module.num_heads, module.num_kv_heads, module.head_size+ if query.dtype != torch.bfloat16 or key.dtype != torch.bfloat16 or heads % kv_heads:+ raise ValueError("tau calibration requires BF16 Q/K and grouped-query attention")+ query = query[:self.tokens].reshape(self.tokens, heads, dim)+ key = key[:self.tokens].reshape(self.tokens, kv_heads, dim).float()+ old_tf32 = torch.backends.cuda.matmul.allow_tf32+ try:+ torch.backends.cuda.matmul.allow_tf32 = False+ for position in self.positions:+ selected = query[position].float().reshape(kv_heads, heads // kv_heads, dim)+ scores = torch.einsum("hrd,thd->hrt", selected, key[:position + 1]).reshape(heads, position + 1)+ stats = online_probability_scale_statistics(+ scores, fp4_tokens=(position // 128) * 128, head_dim=dim,+ probability_multiplier=self.multiplier,+ )+ target = self.samples[position]["codes"]+ for code, bucket in stats["by_amax_e4m3_code"].items():+ merged = target.setdefault(code, {"groups": 0, "error_sum": {"2": 0.0, "6": 0.0}})+ merged["groups"] += bucket["groups"]+ for divisor in ("2", "6"):+ merged["error_sum"][divisor] += bucket["error_sum"][divisor]+ finally:+ torch.backends.cuda.matmul.allow_tf32 = old_tf32+ self.seen.add(id(module))++ def finish(self) -> list[dict]:+ if len(self.seen) != self.expected_layers:+ raise RuntimeError("not every attention layer supplied probability calibration samples")+ return list(self.samples.values())++ def close(self) -> None:+ for handle in self.handles:+ handle.remove()+ self.handles.clear()diff --git a/src/nvfp4_vllm/promote.py b/src/nvfp4_vllm/promote.pynew file mode 100644index 0000000..e8158a5--- /dev/null+++ b/src/nvfp4_vllm/promote.py@@ -0,0 +1,48 @@+"""Sealing a filled BF16 tail page into the FP4 cache.++A sequence's trailing tokens live in BF16 because V is packed along the token+axis a whole page at a time, so a partial page cannot be quantized. The step+that brings a tail to exactly one page removes that obstacle, and the page+moves into the block vLLM already reserved for it. Nothing else in the design+ever writes an FP4 page after a decode step.++One launch per step, always, whether or not any row crossed a boundary. The+alternative — asking the device which rows crossed and launching only then —+costs a synchronization every step to save work on 127 steps out of 128, and+those 128 steps have to pay for the answer anyway. What makes the fixed launch+affordable is that it covers all the layers, K and V at once: one per layer+would cost dozens of launches a step against roughly one millisecond of+attention.++Timing is the whole correctness argument here. This runs after the last+layer's attention, by which point every layer has read the tail as this step's+metadata describes it and written its own share of the token that filled the+page. Running it a layer early would quantize a page that a later layer has+not finished writing, and the loss would be invisible until it showed up as+slightly worse output.+"""++from __future__ import annotations+++def launch(metadata, runtime) -> None:+ """Quantize every filled tail page into its block, for every layer.++ ``promotion_pages`` is the whole slot table rather than this step's rows,+ so the launch shape is a constant of the engine. Rows that did not fill a+ page carry -1 and are skipped inside the kernel.++ The runtime's promotion writer is the write path's kernel with only its+ page roles: K and V pages, no tail, and no rotation or shift, since the+ tail already holds K in the cache's basis. Every layer's own destination+ comes from the base-address table it was built with, because a KV cache is+ allocated a layer at a time and the layers are not a stride apart.+ """+ writer, regions = runtime.promotion_writer()+ writer(+ runtime.tail_key_tokens,+ runtime.tail_value_tokens,+ regions,+ metadata.promotion_source_tokens,+ metadata.promotion_pages,+ )diff --git a/src/nvfp4_vllm/rotation.py b/src/nvfp4_vllm/rotation.pynew file mode 100644index 0000000..dffe37c--- /dev/null+++ b/src/nvfp4_vllm/rotation.py@@ -0,0 +1,59 @@+"""Orthogonal basis changes shared by NVFP4 query and key operands."""++from __future__ import annotations++import math++import torch++BLOCK = 64+++def hadamard(dim: int, device: torch.device | str) -> torch.Tensor:+ """Return the normalized Sylvester Hadamard matrix of order ``dim``.++ The construction intentionally matches the offline rotation search. The+ canonical matrix is FP32; serving may keep a lower-precision execution+ view so Q and K remain in the BF16 contract their quantizers require.+ """+ if type(dim) is not int or dim <= 0 or dim & (dim - 1):+ raise ValueError(+ f"Hadamard dimension must be a positive power of two, got {dim!r}"+ )++ matrix = torch.ones(1, 1, dtype=torch.float32, device=device)+ while matrix.shape[0] < dim:+ matrix = torch.cat(+ [+ torch.cat([matrix, matrix], dim=1),+ torch.cat([matrix, -matrix], dim=1),+ ],+ dim=0,+ )+ return matrix / math.sqrt(dim)+++def block_hadamard_64(+ dim: int, device: torch.device | str+) -> torch.Tensor:+ """Hadamard rotations confined to the active ``roped64`` blocks.++ Each block groups channels with their RoPE partners, then applies a+ normalized H64 matrix within that group.+ """+ if type(dim) is not int or dim <= 0 or dim % BLOCK:+ raise ValueError(+ f"block_hadamard_64 needs a positive multiple of {BLOCK}, got "+ f"{dim!r}"+ )+ low = torch.arange(dim // 2, device=device).reshape(-1, BLOCK // 2)+ blocks = torch.cat((low, low + dim // 2), dim=1)+ if blocks.shape != (dim // BLOCK, BLOCK):+ raise ValueError(+ f"roped64 blocking for dim={dim} has shape {tuple(blocks.shape)}"+ )+ local = hadamard(BLOCK, device)+ rotation = torch.zeros(dim, dim, dtype=torch.float32, device=device)+ for channels in blocks:+ rotation[channels[:, None], channels[None, :]] = local+ return rotationdiff --git a/src/nvfp4_vllm/runtime.py b/src/nvfp4_vllm/runtime.pynew file mode 100644index 0000000..4d973f5--- /dev/null+++ b/src/nvfp4_vllm/runtime.py@@ -0,0 +1,647 @@+"""GPU state one model's NVFP4 layers share for their whole lifetime.++Three things have to outlive a step and be shared by every layer: the BF16+buffer holding each sequence's permanent sink and partial rolling page, the+views that reinterpret a layer's block array as NVFP4 regions, and the buffers+a layer quantizes its query into. The metadata builder cannot own them, because+it has no reference to the attention layers, so they are attached to the+layers and created the first time one of them runs.++That first time is deliberately the profile run, before the KV cache exists.+vLLM sizes the cache from whatever memory is still free once profiling ends, so+anything allocated later would come out of memory already promised to the+cache. Allocating during profiling makes the cache exactly that much smaller,+which is the honest accounting.++Carving the cache views cannot happen that early, since during profiling the+cache is a placeholder with no storage, so the views appear on the first real+write instead.+"""++from __future__ import annotations++import os++import torch+from nvfp4_decode_kernel import RESIDUAL_ROW_TILE, split_partial_count+from nvfp4_decode_kernel.quantize_kv_kernel import KVCacheWriter+from nvfp4_decode_kernel.scale_mode import DEFAULT_SCALE_MODE, sf_vec_size++from . import layout+from .builder import decode_num_splits+from .control import SINK_TOKENS, TAIL_BUFFER_TOKENS+from .rotation import block_hadamard_64, hadamard+++PAGE_SIZE = 128++# Workers load the key-smoothing mean from this safetensors path. Its tensor+# is stored as ``key_shift`` with shape ``[num_layers, num_kv_heads, head_dim]``.+# When unset, no saved mean is loaded.+KEY_SHIFT_ENV = "NVFP4_KEY_SHIFT"+++def load_key_shift(+ num_layers: int,+ num_kv_heads: int,+ head_dim: int,+ device: torch.device,+ dtype: torch.dtype,+ *,+ rotation: torch.Tensor | None = None,+ tensor_parallel_size: int = 1,+ tensor_parallel_rank: int = 0,+) -> torch.Tensor | None:+ """A constant subtracted from post-RoPE K on its way into the cache.++ Post-RoPE K carries large per-channel offsets, and those offsets are what+ sets the group maximum that the block scale is derived from. Removing them+ before quantizing shrinks the scale, so every element in the group gets a+ finer step.++ A full-model sidecar is sliced along KV heads for tensor parallelism.+ Already rank-local sidecars keep their shape. Head replication is not+ inferred: only exactly ``num_kv_heads * tensor_parallel_size`` full-model+ heads are accepted.++ The sidecar is stored in the original basis. If ``rotation`` is present,+ this loader changes the constant to ``c R^T`` once, in float64, and rounds+ it once into ``dtype``.++ Nothing adds the constant back. Attention scores become ``q·k - q·mu``,+ and ``q·mu`` is the same for every key a given query sees, so softmax is+ unchanged. That argument only holds if *every* stored key is shifted, which+ is why the subtraction sits at the single point all three write paths pass+ through rather than inside the quantizer.++ Rounding ``mu`` was the obvious thing to be afraid of and measures as+ nothing: the invariance above holds for any constant, however coarse, and+ against captured post-RoPE K the bfloat16 constant removes 55.80% of the+ NVFP4 error where the float32 one removes 55.75%. The step it is rounded+ against is four-bit; a 2e-3 relative error on the constant disappears into+ it. So ``dtype`` is whatever the write path subtracts in for free: the+ cache's own for a shift applied as one BF16 ``sub``, float32 where the K+ writer already works in float32.+ """+ path = os.environ.get(KEY_SHIFT_ENV)+ if not path:+ return None+ if tensor_parallel_size < 1 or not 0 <= tensor_parallel_rank < tensor_parallel_size:+ raise ValueError(+ "invalid tensor-parallel coordinates: "+ f"rank={tensor_parallel_rank}, size={tensor_parallel_size}"+ )++ from safetensors.torch import load_file++ tensors = load_file(path)+ if "key_shift" not in tensors:+ raise ValueError(+ f"{path} has no 'key_shift' tensor, found {sorted(tensors)}"+ )+ shift = tensors["key_shift"]+ expected = (num_layers, num_kv_heads, head_dim)+ global_shape = (num_layers, num_kv_heads * tensor_parallel_size, head_dim)+ if tensor_parallel_size > 1 and tuple(shift.shape) == global_shape:+ # vLLM partitions unreplicated KV heads in contiguous rank order.+ start = tensor_parallel_rank * num_kv_heads+ shift = shift[:, start:start + num_kv_heads, :].contiguous()+ if tuple(shift.shape) != expected:+ raise ValueError(+ f"{path} holds a key_shift of {tuple(shift.shape)}, this model "+ f"needs {expected} (rank-local) or {global_shape} (full-model)"+ )+ return _active_shift(shift.to(device), rotation, dtype)+++def _active_shift(+ shift: torch.Tensor, rotation: torch.Tensor | None, dtype: torch.dtype+) -> torch.Tensor:+ """An original-basis mean as the write path subtracts it, rounded once."""+ if rotation is None:+ return shift.to(dtype)+ exact = torch.matmul(+ shift.double(), rotation.to(device=shift.device).double().T+ )+ return exact.to(dtype)+++class LayerRuntime:+ """The tail buffers, cache views, and query scratch one model shares."""++ def __init__(+ self,+ *,+ num_layers: int,+ num_slots: int,+ num_heads: int,+ num_kv_heads: int,+ head_dim: int,+ device: torch.device,+ query_terms: int = 1,+ rotation: str = "none",+ key_scale_mode: str = DEFAULT_SCALE_MODE,+ value_scale_mode: str = DEFAULT_SCALE_MODE,+ key_scale_geometry: str = "local",+ ) -> None:+ if query_terms not in (1, 2):+ raise ValueError(f"query_terms must be 1 or 2, got {query_terms!r}")+ if rotation not in ("none", "hadamard", "block_hadamard_64"):+ raise ValueError(+ "rotation must be 'none', 'hadamard', or "+ f"'block_hadamard_64', got {rotation!r}"+ )+ self.num_layers = num_layers+ self.num_kv_heads = num_kv_heads+ self.head_dim = head_dim+ self.query_terms = query_terms+ self.rotation_name = rotation+ self.key_scale_mode = key_scale_mode+ self.value_scale_mode = value_scale_mode+ if key_scale_geometry not in ("local", "gaussian_global"):+ raise ValueError(+ "key_scale_geometry must be 'local' or 'gaussian_global', "+ f"got {key_scale_geometry!r}"+ )+ self.key_scale_geometry = key_scale_geometry+ rotations = {+ "none": None,+ "hadamard": hadamard,+ "block_hadamard_64": block_hadamard_64,+ }+ constructor = rotations[rotation]+ self.rotation = (+ None if constructor is None else constructor(head_dim, device)+ )+ # ``hadamard`` is applied inside the query quantizer and the K writer,+ # in FP32 butterflies; any other rotation stays a BF16 GEMM, with this+ # execution view kept so that no layer pays a cast and an allocation+ # for it. The FP32 matrix remains canonical for sidecar transforms.+ self.fused_rotation = rotation == "hadamard"+ self._rotation_by_dtype: dict[torch.dtype, torch.Tensor] = {}+ if self.rotation is not None:+ self._rotation_by_dtype = {torch.float32: self.rotation}+ if not self.fused_rotation:+ self._rotation_by_dtype[torch.bfloat16] = self.rotation.to(+ torch.bfloat16+ )+ # Held separately from the vector so that turning the shift off keeps+ # what it cost to measure. A/B-ing the two arms in one process is the+ # reason to want that.+ self.key_shift_enabled = True+ # Set only while measuring a shift, which cannot happen at the same+ # time as applying one: the sum has to be taken over the K the model+ # actually produces. Each layer accumulates its own row; the token+ # count is shared, since every layer sees the same tokens.+ self.key_shift_sum: torch.Tensor | None = None+ self.key_shift_tokens = 0++ # One allocation each, indexed by layer, so promotion can take the+ # layer as a grid dimension instead of looping in Python. Zeroed+ # rather than empty because attention reads a whole tail page back and+ # the positions past the sequence must not be denormals or NaNs.+ shape = (+ num_layers,+ num_slots,+ TAIL_BUFFER_TOKENS,+ num_kv_heads,+ head_dim,+ )+ self.tail_key = torch.zeros(shape, dtype=torch.bfloat16, device=device)+ self.tail_value = torch.zeros(+ shape, dtype=torch.bfloat16, device=device+ )+ # The same storage seen as one run of tokens, which is what promotion+ # quantizes out of: layer ``l`` slot ``s`` starts at token+ # ``l * num_slots * TAIL_BUFFER_TOKENS+ # + s * TAIL_BUFFER_TOKENS``.+ # Loaded once the tail exists, because the shift is subtracted from K+ # on its way into that tail and carrying a second dtype would put a+ # cast in the write path.+ tp_size, tp_rank = 1, 0+ if os.environ.get(KEY_SHIFT_ENV) and torch.distributed.is_initialized():+ from vllm.distributed import (+ get_tensor_model_parallel_rank,+ get_tensor_model_parallel_world_size,+ )++ tp_size = get_tensor_model_parallel_world_size()+ tp_rank = get_tensor_model_parallel_rank()+ self.key_shift = load_key_shift(+ num_layers,+ num_kv_heads,+ head_dim,+ device,+ self.key_shift_dtype,+ rotation=self.rotation,+ tensor_parallel_size=tp_size,+ tensor_parallel_rank=tp_rank,+ )+ self.tail_key_tokens = self.tail_key.view(-1, num_kv_heads, head_dim)+ self.tail_value_tokens = self.tail_value.view(+ -1, num_kv_heads, head_dim+ )++ # Zeroed once here and never again, which is the contract fp4_decode+ # asks of a caller-owned buffer: the quantizer writes row 0 of each+ # tile and nothing writes the rest. Shared by every layer, since a+ # layer is done with it before the next one runs.+ self.query_padded = torch.zeros(+ num_slots,+ RESIDUAL_ROW_TILE,+ num_heads,+ head_dim,+ dtype=torch.bfloat16,+ device=device,+ )+ # The quantizer's two outputs, sized and shared the same way, so that+ # quantizing a query allocates and fills nothing. The packed query is+ # rewritten whole every call and what it holds now does not matter.+ # The scales need the zeroing more than query_padded does: the layout+ # gives each KV head 32 * 4 row slots and each four-scale k-atom a+ # four-byte scale word, and only the num_heads // num_kv_heads slots+ # that carry a query head are ever written. The guards keep Q in the+ # format K is in, so K's mode sizes the k-atoms.+ self.query_fp4 = torch.zeros(+ num_slots,+ 1,+ num_heads,+ head_dim // 2,+ dtype=torch.uint8,+ device=device,+ )+ self.query_scales = torch.zeros(+ num_slots,+ 1,+ num_heads,+ head_dim // (4 * sf_vec_size(key_scale_mode)),+ 32,+ 4,+ 4,+ dtype=torch.uint8,+ device=device,+ )+ self.query_fp4_second = (+ torch.zeros_like(self.query_fp4) if query_terms == 2 else None+ )+ self.query_scales_second = (+ torch.zeros_like(self.query_scales) if query_terms == 2 else None+ )+ # The split path's partial outputs and LSEs, sized for the widest+ # split launch any decode batch this engine can form asks for, and+ # shared like the query scratch. fp4_decode needs them neither zeroed+ # nor refilled, so empty is the honest initial state. A runtime on the+ # CPU runs no decode kernel and gets none.+ sms = (+ torch.cuda.get_device_properties(device).multi_processor_count+ if torch.device(device).type == "cuda"+ else 0+ )+ # Every decode carries the BF16 tail, so a split launch may hold one+ # more partial per row than it has splits.+ split_rows = max(+ (+ rows+ * split_partial_count(+ rows, num_kv_heads, splits, residual=True, sms=sms+ )+ for rows in range(1, num_slots + 1)+ if (splits := decode_num_splits(rows, num_kv_heads, sms)) > 1+ ),+ default=0,+ )+ self.split_partial = (+ torch.empty(+ split_rows * num_heads * head_dim,+ dtype=torch.float32,+ device=device,+ )+ if split_rows+ else None+ )+ self.split_lse = (+ torch.empty(split_rows * num_heads, dtype=torch.float32, device=device)+ if split_rows+ else None+ )++ self._views: dict[int, tuple[torch.Tensor, ...]] = {}+ self._by_layer: dict[int, tuple[torch.Tensor, ...]] = {}+ self._bases: torch.Tensor | None = None+ self._expected_layer = 0+ self._writer: KVCacheWriter | None = None+ self._writer_inputs: tuple = ()+ self._bound: dict[int, tuple] = {}+ self._promotion: tuple | None = None+ self._promotion_inputs: tuple = ()++ def rotation_for(self, dtype: torch.dtype) -> torch.Tensor | None:+ """The shared rotation in an operand's dtype, allocated only once."""+ if self.rotation is None:+ return None+ matrix = self._rotation_by_dtype.get(dtype)+ if matrix is None:+ matrix = self.rotation.to(dtype=dtype)+ self._rotation_by_dtype[dtype] = matrix+ return matrix++ @property+ def key_shift_dtype(self) -> torch.dtype:+ """What the shift is held in: see :func:`load_key_shift`."""+ return torch.float32 if self.fused_rotation else self.tail_key.dtype++ @property+ def query_rotation(self) -> str:+ """The rotation ``fp4_decode``'s quantizer applies to the BF16 query."""+ return "hadamard" if self.fused_rotation else "none"++ def transform_query(self, query: torch.Tensor) -> torch.Tensor:+ """The part of Q's basis change the quantizer does not do itself."""+ if self.rotation is None or self.fused_rotation:+ return query+ return torch.matmul(query, self.rotation_for(query.dtype).T)++ def key_write(+ self, layer_index: int, key: torch.Tensor+ ) -> tuple[torch.Tensor, bool, torch.Tensor | None]:+ """The K a cache write takes and what it still has to apply to it.++ The fused rotation and every shift are left to the writers, which do+ them in the same pass that copies K; only an unfused rotation is+ applied here, as a GEMM.+ """+ shift = self.active_key_shift+ if self.rotation is not None and not self.fused_rotation:+ return self.transform_key_for_cache(layer_index, key), False, None+ return (+ key,+ self.fused_rotation,+ None if shift is None else shift[layer_index],+ )++ @property+ def decode_writes_tail(self) -> bool:+ """Whether the decode prefix's quantizer writes its own tail tokens.++ It does whenever the writer would have rotated (or not) and shifted+ the raw K itself, with an FP32 shift; an unfused rotation hands the+ writer an already rotated K that the decode path never sees.+ """+ shift = self.active_key_shift+ return (self.rotation is None or self.fused_rotation) and (+ shift is None or shift.dtype == torch.float32+ )++ def writer_for(+ self, layer_index: int, kv_cache: torch.Tensor+ ) -> tuple[KVCacheWriter, tuple]:+ """The write path's writer and this layer's regions, bound for it.++ Built on first use and again whenever the shift+ in force changes, since the writer holds it; a layer's regions are+ checked and bound once, when its cache is first carved.+ """+ bound = self._bound.get(layer_index)+ if bound is None or bound[0] is not kv_cache:+ carved = self.views(layer_index, kv_cache)+ writer = self._current_writer(carved)+ bound = (kv_cache, writer.bind_regions(*carved))+ self._bound[layer_index] = bound+ return self._current_writer(None), bound[1]++ def _current_writer(self, carved) -> KVCacheWriter:+ unfused = self.rotation is not None and not self.fused_rotation+ inputs = (+ None if unfused else self.active_key_shift,+ )+ writer = self._writer+ if writer is not None and all(+ a is b for a, b in zip(inputs, self._writer_inputs)+ ):+ return writer+ (shift,) = inputs+ writer = KVCacheWriter(+ regions=carved if carved is not None else self._by_layer[min(self._by_layer)],+ key_scale_mode=self.key_scale_mode,+ value_scale_mode=self.value_scale_mode,+ key_scale_geometry=self.key_scale_geometry,+ rotate_key=self.fused_rotation,+ key_shift=shift,+ tail_key=self.tail_key,+ tail_value=self.tail_value,+ sink_tokens=SINK_TOKENS,+ )+ self._writer, self._writer_inputs = writer, inputs+ return writer++ def transform_key_for_cache(+ self, layer_index: int, key: torch.Tensor+ ) -> torch.Tensor:+ """Cache-bound K in the cache's basis, as a tensor of its own.++ Only for what the writers do not fuse: a shift alone, or a rotation+ applied as a GEMM. The fused Hadamard exists only inside the writers.+ """+ if self.fused_rotation:+ raise ValueError(+ "the fused Hadamard is applied by the cache writers, not here"+ )+ shift = self.active_key_shift+ rotation = self.rotation_for(key.dtype)+ if rotation is None:+ return key if shift is None else torch.sub(key, shift[layer_index])+ if shift is None:+ return torch.matmul(key, rotation.T)++ # baddbmm broadcasts one per-head shift across all tokens and fuses+ # ``k R^T - c R^T`` into the GEMM epilogue.+ return torch.baddbmm(+ shift[layer_index],+ key,+ rotation.T.expand(key.shape[0], -1, -1),+ beta=-1.0,+ )++ @property+ def active_key_shift(self) -> torch.Tensor | None:+ """What the write path should subtract, or None to store K as it is."""+ return self.key_shift if self.key_shift_enabled else None++ def set_key_shift_enabled(self, enabled: bool) -> None:+ """Turn the shift on or off between requests.++ Only between them. Keys already in the cache were stored under+ whichever setting was in force when they were written, so a sequence+ that spans a flip has two different constants in one softmax. The+ caller has to know the cache is drained; nothing here can check it.+ """+ if enabled and self.key_shift is None:+ raise ValueError(+ "no shift has been loaded or measured, so there is nothing "+ "to enable"+ )+ self.key_shift_enabled = enabled++ def begin_key_shift_measurement(self) -> None:+ """Measure the K mean instead of removing it, until told to stop.++ Any shift in force is dropped first. Measuring against a shifted K+ would give the residual mean rather than the mean, and the two are not+ interchangeable: the second measurement would be near zero and would+ look like there was nothing to remove.+ """+ self.key_shift = None+ self.key_shift_sum = torch.zeros(+ self.num_layers,+ self.num_kv_heads,+ self.head_dim,+ dtype=torch.float64,+ device=self.tail_key.device,+ )+ self.key_shift_tokens = 0++ def finish_key_shift_measurement(self) -> torch.Tensor:+ """Freeze what was measured and start subtracting it.++ Accumulated in float64. The returned value is the mean rounded once+ into the cache's dtype, in the original basis, for sidecar storage; the+ active value is the same mean rotated in float64 and rounded once into+ :attr:`key_shift_dtype`.+ """+ if self.key_shift_sum is None or self.key_shift_tokens == 0:+ raise ValueError(+ "no tokens were measured, so there is no mean to subtract; "+ "begin_key_shift_measurement must precede a forward pass"+ )+ mean = self.key_shift_sum / self.key_shift_tokens+ measured = mean.to(self.tail_key.dtype)+ self.key_shift = _active_shift(+ measured if self.rotation is None else mean,+ self.rotation,+ self.key_shift_dtype,+ )+ self.key_shift_enabled = True+ self.key_shift_sum = None+ self.key_shift_tokens = 0+ # The sidecar contract is always the model's original post-RoPE basis.+ return measured++ def views(+ self, layer_index: int, kv_cache: torch.Tensor+ ) -> tuple[torch.Tensor, ...]:+ """The four NVFP4 regions of one layer's block array.++ Carving is only address arithmetic, but it builds Python objects and+ this runs once per layer per step, so the result is kept. A layer's+ cache tensor is allocated once and never moves, which is what makes its+ address usable as the key.++ The layer index is recorded alongside, because promotion needs every+ layer's address in one table and this is the only place that learns+ which cache a layer actually uses. Taking it from here rather than+ from vLLM's module registry means promotion writes into the tensor the+ write path wrote into, under the index the write path used, without+ either of those having to be what vLLM believes.+ """+ key = kv_cache.data_ptr()+ carved = self._views.get(key)+ if carved is None:+ carved = layout.carve(+ kv_cache, self.num_kv_heads, self.head_dim, self.key_scale_mode+ )+ self._views[key] = carved+ if self._by_layer.get(layer_index) is not carved:+ self._by_layer[layer_index] = carved+ self._bases = None+ return carved++ @property+ def destination_bases(self) -> torch.Tensor:+ """``[4, num_layers]`` int64. Where each layer's four regions start.++ Built on first use and kept, because it is a host-to-device copy and+ promotion runs every step. Every layer must have written its cache by+ then, which the execution order below guarantees: promotion fires+ after the last layer, and each one carves its cache before it writes.+ """+ if self._bases is None:+ missing = [+ index+ for index in range(self.num_layers)+ if index not in self._by_layer+ ]+ if missing:+ raise ValueError(+ f"layers {missing} have not written their KV cache, so "+ "promotion has no address to send their pages to"+ )+ self._bases = torch.tensor(+ [+ [+ self._by_layer[layer][region].data_ptr()+ for layer in range(self.num_layers)+ ]+ for region in range(4)+ ],+ dtype=torch.int64,+ device=self.tail_key.device,+ )+ return self._bases++ def promotion_writer(self) -> tuple[KVCacheWriter, tuple]:+ """The writer that seals filled tail pages for every layer at once.++ The write path's kernel with its page roles only, over the base table:+ the tail already holds K in the cache's basis, so it neither rotates+ nor shifts. Built on first use and again when a layer's cache is+ carved anew (a new table).+ Layer 0's regions give the layout; the table gives every address.+ """+ inputs = (+ self.destination_bases,+ )+ promotion = self._promotion+ if promotion is not None and all(+ a is b for a, b in zip(inputs, self._promotion_inputs)+ ):+ return promotion+ (bases,) = inputs+ regions = self._by_layer[0]+ writer = KVCacheWriter(+ regions=regions,+ key_scale_mode=self.key_scale_mode,+ value_scale_mode=self.value_scale_mode,+ key_scale_geometry=self.key_scale_geometry,+ destination_bases=bases,+ )+ self._promotion = (writer, writer.bind_regions(*regions))+ self._promotion_inputs = inputs+ return self._promotion++ def expect_layer(self, layer_index: int) -> None:+ """Refuse a step whose layers do not run in the order they are indexed.++ Promotion fires after the highest index, which is the last layer only+ if construction order is execution order. That holds for a decoder+ stack, and nothing in vLLM promises it. If it stopped holding, a page+ would be sealed into FP4 before some layer had written its share of+ the boundary token, and what reached the user would be a model that is+ slightly and inexplicably worse. Two integers a layer is cheaper than+ ever having to debug that.++ The count carries across steps rather than resetting at layer 0, so+ that a step which ran only some of its layers is caught by the next+ step's first layer arriving early. It resynchronizes when it complains,+ because the other way to leave it mid-stack is for something else to+ raise partway through a step, and a stale count would then answer every+ later step with a complaint about layer order instead of about whatever+ actually broke.+ """+ if layer_index != self._expected_layer:+ expected = self._expected_layer+ self._expected_layer = (layer_index + 1) % self.num_layers+ raise ValueError(+ f"layer {layer_index} ran where layer {expected} was expected; "+ "promotion assumes layers run in index order"+ )+ self._expected_layer = (layer_index + 1) % self.num_layersdiff --git a/src/nvfp4_vllm/write.py b/src/nvfp4_vllm/write.pynew file mode 100644index 0000000..f208a86--- /dev/null+++ b/src/nvfp4_vllm/write.py@@ -0,0 +1,275 @@+"""Placing this step's K/V into the NVFP4 cache and the BF16 tail.++The first 16 tokens are copied into the permanent BF16 sink while the ordinary+vLLM page grid is preserved. Until page zero fills, those tokens are also in+the rolling BF16 page; after promotion they have an FP4 copy that decode masks.+The duplication avoids moving every later page boundary, which would change+page-local FP4-P rounding. The final partial page is copied into the rolling+BF16 tail, and the two BF16 regions share one slot.++Which tokens fall on which side is decided entirely on the device. The work+table and the layer's write (a ``KVCacheWriter`` launch) read the control+plane's per-row answers and the block table, so no length or offset is ever+brought back to the host, and every launch shape comes from host integers vLLM+already has.+"""++from __future__ import annotations++import torch+import triton+import triton.language as tl++from nvfp4_decode_kernel.quantize_kv_kernel import KVCacheWriter++from .control import PAGE_SIZE+++def _work_table_kernel(+ query_start_loc_ptr,+ seqused_fp4_ptr,+ row_to_slot_ptr,+ block_table_ptr,+ block_table_stride,+ source_tokens_ptr,+ destination_pages_ptr,+ num_reqs,+ columns_per_row,+ PAGE: tl.constexpr,+ BLOCK: tl.constexpr,+):+ """Say, for each slot of the quantizer's grid, what to read and where.++ The grid is a fixed rectangle of rows by page columns, so most slots in a+ given step do nothing; those get a destination of -1, which the quantizer+ treats as no work. A row contributes pages only if it is a fresh prefill,+ because only then are the tokens of its full pages present in this step's+ activations. A decode row's earlier pages were quantized when they were+ written and must not be touched again.+ """+ row = tl.program_id(0)+ columns = tl.arange(0, BLOCK)+ within = columns < columns_per_row++ start = tl.load(query_start_loc_ptr + row)+ end = tl.load(query_start_loc_ptr + row + 1)+ slot = tl.load(row_to_slot_ptr + row)+ full_pages = tl.load(seqused_fp4_ptr + row) // PAGE++ row_writes = (end - start > 1) & (slot >= 0)+ active = within & row_writes & (columns < full_pages)++ blocks = tl.load(+ block_table_ptr + row * block_table_stride + columns,+ mask=active,+ other=-1,+ )+ out = row * columns_per_row + columns+ tl.store(+ source_tokens_ptr + out,+ start + columns * PAGE,+ mask=within,+ )+ tl.store(+ destination_pages_ptr + out,+ tl.where(active, blocks, -1),+ mask=within,+ )+++def _tail_reset_kernel(+ tail_key_ptr,+ tail_value_ptr,+ tail_layer_stride,+ tail_slot_stride,+ query_start_loc_ptr,+ seq_lens_ptr,+ row_to_slot_ptr,+ num_reqs,+ WIDTH: tl.constexpr,+ TOKENS: tl.constexpr,+):+ """Clear the tail page a brand new request is about to move into.++ Attention reads a row's whole tail page and masks the positions past its+ length by weighting them zero, so those positions have to hold a finite+ number: zero times a NaN is a NaN, and one poisoned lane takes the row's+ entire output with it. The buffer is allocated zeroed for that reason, but+ a slot outlives the request that filled it, and the next occupant inherits+ whatever the last one left past its own length. Clearing on arrival is what+ keeps the allocation-time invariant true for every later tenant.++ A request is on its first step when the sequence is no longer than what it+ brought, which catches a single-token prompt that the batch reordering+ filed among the decodes, and a recycled block id that let a new request+ match some earlier tenant's slot. Both keep a stale page otherwise.+ """+ layer = tl.program_id(0)+ row = tl.program_id(1)++ slot = tl.load(row_to_slot_ptr + row, mask=row < num_reqs, other=-1)+ start = tl.load(query_start_loc_ptr + row, mask=row < num_reqs, other=0)+ end = tl.load(query_start_loc_ptr + row + 1, mask=row < num_reqs, other=0)+ seq_len = tl.load(seq_lens_ptr + row, mask=row < num_reqs, other=0)++ if (slot >= 0) & (seq_len <= end - start):+ lanes = tl.arange(0, WIDTH)+ zero = tl.zeros([WIDTH], dtype=tail_key_ptr.dtype.element_ty)+ for position in tl.range(0, TOKENS):+ destination = (+ layer * tail_layer_stride+ + slot * tail_slot_stride+ + position * WIDTH+ + lanes+ )+ tl.store(tail_key_ptr + destination, zero)+ tl.store(tail_value_ptr + destination, zero)+++def reset_new_request_tails(+ *,+ tail_key: torch.Tensor,+ tail_value: torch.Tensor,+ query_start_loc: torch.Tensor,+ seq_lens: torch.Tensor,+ row_to_slot: torch.Tensor,+) -> None:+ """Clear every layer's tail page for the rows starting a request this step.++ Takes the tail for all layers at once and runs before any of them writes,+ rather than once per layer, because a slot's history has to end for the+ whole model at the same moment; a later layer clearing its own page would+ also have to do it after that layer had already read one.++ The grid covers every row whether or not it is new, since which rows are+ new is a device-side answer and asking the host would cost the very+ synchronization the control plane exists to avoid. One program owns a+ whole page so a row that is not new exits once instead of once per token;+ a new row clears the page in an inner loop.+ """+ num_layers, _, tokens, heads, head_dim = tail_key.shape+ _tail_reset_kernel[(num_layers, seq_lens.shape[0])](+ tail_key,+ tail_value,+ tail_key.stride(0),+ tail_key.stride(1),+ query_start_loc,+ seq_lens,+ row_to_slot,+ seq_lens.shape[0],+ WIDTH=heads * head_dim,+ TOKENS=tokens,+ num_warps=4,+ )+++class PageWorkTable:+ """Per-step description of which full pages this step must quantize.++ Built once per step by the metadata builder and read by every layer. It is+ a dense rectangle rather than a compacted list because compacting would+ need a count the host does not have, and the rectangle is small: rows times+ pages per row, both bounded by the scheduler's limits.+ """++ def __init__(+ self, *, max_num_seqs: int, max_num_batched_tokens: int, device+ ) -> None:+ self.max_num_seqs = max_num_seqs+ # Chunked prefill is refused, so a prefill row's whole prompt arrives+ # in one step and its full pages cannot outnumber the step's tokens.+ self.max_columns = max_num_batched_tokens // PAGE_SIZE + 1+ capacity = max_num_seqs * self.max_columns+ self.source_tokens = torch.empty(+ capacity, dtype=torch.int32, device=device+ )+ self.destination_pages = torch.full(+ (capacity,), -1, dtype=torch.int32, device=device+ )++ def build(+ self,+ *,+ query_start_loc: torch.Tensor,+ seqused_fp4: torch.Tensor,+ row_to_slot: torch.Tensor,+ block_table: torch.Tensor,+ num_reqs: int,+ max_query_len: int,+ ) -> tuple[torch.Tensor, torch.Tensor]:+ # A step where no row runs more than one token has no full pages to+ # write, and an empty table leaves every layer's write without page+ # CTAs, or without a launch when no tail token is its to write either.+ if max_query_len <= 1:+ return self.source_tokens[:0], self.destination_pages[:0]++ columns = min(+ max_query_len // PAGE_SIZE + 1,+ self.max_columns,+ block_table.shape[1],+ )+ items = num_reqs * columns+ _work_table_kernel[(num_reqs,)](+ query_start_loc,+ seqused_fp4,+ row_to_slot,+ block_table,+ block_table.stride(0),+ self.source_tokens,+ self.destination_pages,+ num_reqs,+ columns,+ PAGE=PAGE_SIZE,+ BLOCK=triton.next_power_of_2(columns),+ num_warps=4,+ )+ return self.source_tokens[:items], self.destination_pages[:items]+++def write_kv(+ *,+ writer: KVCacheWriter,+ regions: tuple,+ key: torch.Tensor,+ value: torch.Tensor,+ source_tokens: torch.Tensor,+ destination_pages: torch.Tensor,+ query_start_loc: torch.Tensor,+ seq_lens: torch.Tensor,+ seqused_fp4: torch.Tensor,+ row_to_slot: torch.Tensor,+ layer_index: int,+ decode_rows: int,+ first_row: int,+) -> None:+ """Place one layer's K/V for this step: full pages quantized, rest tailed.++ One launch of the runtime's writer (see ``LayerRuntime.writer_for``) into+ this layer's ``regions`` and its slice of the shared tail: rows+ ``[:128]`` of a slot are the rolling tail and rows ``[128:144]`` the+ permanent sink. ``key`` is what the writer puts into the cache's basis on+ the way in, which it does for the pages and the tail alike, so a key+ reaches the cache with the same bytes whichever of them stores it.++ ``key`` and ``value`` may run past this step's tokens (vLLM pads them);+ the writer reads only the tokens the work table and the rows name. Rows+ before ``first_row`` are left out of the tail: the decode prefix passes+ its own tokens to the query quantizer instead, which writes them with the+ same arithmetic in its own launch.+ """+ writer(+ key,+ value,+ regions,+ source_tokens,+ destination_pages,+ query_start_loc,+ seq_lens,+ seqused_fp4,+ row_to_slot,+ layer_index,+ decode_rows,+ first_row,+ )diff --git a/third_party/vllm b/third_party/vllmnew file mode 160000index 0000000..1ad84fe--- /dev/null+++ b/third_party/vllm@@ -0,0 +1 @@+Subproject commit 1ad84fea866bc478942efa8550036ffa52a51283diff --git a/tools/bootstrap_env.sh b/tools/bootstrap_env.shnew file mode 100755index 0000000..d55c8c2--- /dev/null+++ b/tools/bootstrap_env.sh@@ -0,0 +1,46 @@+#!/usr/bin/env bash+# Bootstrap the BitDecoding-FP4 inference environment.+#+# Usage:+# bash tools/bootstrap_env.sh+# bash tools/bootstrap_env.sh _local/envs/bd_fp4+set -euo pipefail++REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"+REPO_ROOT="$(realpath "${REPO_ROOT}")"+export REPO_ROOT+cd "${REPO_ROOT}"++# Resolved so that a symlink to local disk is installed through its target:+# through the link, every file operation is a lookup on the repo's mount.+venv="$(realpath -m "${1:-_local/envs/bd_fp4}")"+export UV_LINK_MODE=copy++if ! command -v uv >/dev/null 2>&1; then+ echo "Install uv first: python3.12 -m pip install uv" >&2+ exit 1+fi++uv venv --managed-python --python 3.12 "${venv}"+py="${venv}/bin/python"+uv pip install --python "${py}" \+ torch==2.11.0 torchvision==0.26.0 torchaudio==2.11.0 \+ --index-url https://download.pytorch.org/whl/cu130+uv pip install --python "${py}" \+ "cmake>=3.26.1" ninja "packaging>=24.2" \+ "setuptools>=77.0.3,<81.0.0" setuptools-scm setuptools-rust maturin wheel jinja2+(+ cd third_party/vllm+ VLLM_USE_PRECOMPILED=1 uv pip install \+ --python "${py}" -e . --no-build-isolation \+ "quack-kernels==0.6.1" \+ --extra-index-url https://flashinfer.ai/whl/ \+ --extra-index-url https://download.pytorch.org/whl/cu130 \+ --index-strategy unsafe-best-match+)+uv pip install --python "${py}" -e ".[eval]"+"${py}" -c \+ "import torch, vllm, flashinfer, nvfp4_decode_kernel; print('torch', torch.__version__, '| vllm', vllm.__version__, '| flashinfer', flashinfer.__version__)"+echo "inference environment ready: ${venv}"+echo "LiveCodeBench grading uses this same interpreter in a separate process."+echo "Put a LiveCodeBench checkout on PYTHONPATH (_local/LiveCodeBench); do not pip install it."diff --git a/tools/calibrate_key_smooth_sidecar.py b/tools/calibrate_key_smooth_sidecar.pynew file mode 100644index 0000000..abab091--- /dev/null+++ b/tools/calibrate_key_smooth_sidecar.py@@ -0,0 +1,135 @@+"""Calibrate static key smoothing and optional probability threshold tau.++The accuracy runner invokes this in a separate process before loading the+scoring engine. Tensor-parallel workers export one global key mean, which+serving workers slice when loading the saved calibration.+"""++from __future__ import annotations++import argparse+import json+import os+from pathlib import Path+++def build_parser() -> argparse.ArgumentParser:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument(+ "--model-path", required=True, help="checkpoint to measure the mean on"+ )+ parser.add_argument(+ "--output-path",+ type=Path,+ required=True,+ help="destination for the calibrated key mean",+ )+ parser.add_argument(+ "--max-model-len",+ type=int,+ default=10240,+ help="engine context; match the rollout the shift will serve",+ )+ parser.add_argument(+ "--context-tokens",+ type=int,+ default=32768,+ help=(+ "requested self-generated context; calibration clamps this to "+ "max_model_len - 1"+ ),+ )+ parser.add_argument(+ "--gpu-memory-utilization",+ type=float,+ default=0.85,+ )+ parser.add_argument("--tensor-parallel-size", type=int, default=1)+ parser.add_argument("--max-num-seqs", type=int, default=32)+ parser.add_argument("--probability-multiplier", type=float, default=None,+ help="also calibrate tau during the key-smoothing prefill, using this c")+ parser.add_argument(+ "--scale-mode", default=None,+ help="scale rule, or a JSON mapping with key/value/query rules",+ )+ parser.add_argument("--query-terms", type=int, choices=(1, 2), default=None)+ parser.add_argument("--rotation", choices=("none", "hadamard"), default=None)+ return parser+++def main() -> None:+ args = build_parser().parse_args()+ if args.output_path.exists():+ raise SystemExit(+ f"{args.output_path} exists; a sidecar in use by a run must not be "+ "rewritten underneath it"+ )++ # The measurement has to happen under the engine that will serve it: the+ # mean is post-RoPE, so it depends on the positions visited, and those+ # depend on the window. TP workers export one global-head sidecar that+ # the runtime slices for each serving rank.+ from nvfp4_vllm import build_llm, calibrate_key_smoothing++ scale_mode = args.scale_mode+ if scale_mode is not None and scale_mode.lstrip().startswith("{"):+ scale_mode = json.loads(scale_mode)+ additional_config = {+ key: value for key, value in {+ "nvfp4_scale_mode": scale_mode,+ "nvfp4_query_terms": args.query_terms,+ "nvfp4_rotation": args.rotation,+ }.items() if value is not None+ }+ additional_config["nvfp4_key_smoothing"] = None+ llm = build_llm(+ args.model_path,+ dtype="bfloat16",+ max_model_len=args.max_model_len,+ tensor_parallel_size=args.tensor_parallel_size,+ gpu_memory_utilization=args.gpu_memory_utilization,+ enforce_eager=True,+ trust_remote_code=True,+ max_num_seqs=args.max_num_seqs,+ max_num_batched_tokens=max(40_960, args.max_model_len),+ additional_config=additional_config,+ )++ args.output_path.parent.mkdir(parents=True, exist_ok=True)+ probability = ({"probability_multiplier": args.probability_multiplier}+ if args.probability_multiplier is not None else {})+ measured = calibrate_key_smoothing(+ llm,+ context_tokens=args.context_tokens,+ save_to=str(args.output_path),+ **probability,+ )+ if measured is None:+ raise SystemExit(+ "the engine came up without the NVFP4 cache, so there was no mean "+ "to measure"+ )++ provenance = args.output_path.with_suffix(".json")+ temporary = provenance.with_suffix(".json.tmp")+ temporary.write_text(+ json.dumps(+ {+ "model_path": os.path.realpath(args.model_path),+ "max_model_len": args.max_model_len,+ "context_tokens": args.context_tokens,+ "tensor_parallel_size": args.tensor_parallel_size,+ "additional_config": additional_config,+ **measured,+ },+ indent=2,+ )+ + "\n"+ )+ temporary.replace(provenance)+ print(f"[key smoothing] {measured}", flush=True)+ print(f"[key smoothing] provenance {provenance}", flush=True)+++if __name__ == "__main__":+ main()diff --git a/tools/lcb_data.py b/tools/lcb_data.pynew file mode 100644index 0000000..1c2cb56--- /dev/null+++ b/tools/lcb_data.py@@ -0,0 +1,102 @@+#!/usr/bin/env python3+"""Load the LiveCodeBench v6 slice without paying for the other five.++``load_code_generation_dataset`` cannot fetch one version. Its loader script+calls ``dl_manager.download_and_extract(_URLS)`` on the whole mapping and only+then filters by ``version_tag``, so asking for the 175-problem v6 slice+downloads 4,486 MB to read 134 of it. The slice is exactly ``test6.jsonl``.++Order is the file's own line order, which is what the official loader returns:+its ``_generate_examples`` walks files then lines, and ``load_dataset`` does+not reorder. That order is the contract with the grader, so ``bench_index`` is+the line number and nothing here sorts.++This module imports ``lcb_runner`` from a LiveCodeBench checkout on+``PYTHONPATH`` (``_local/LiveCodeBench``). Do not pip-install LiveCodeBench:+its ``pyproject.toml`` would pull a different vLLM. Grading runs in a **new+process** of the same inference environment so it does not share a CUDA+context with the engine.+"""+from __future__ import annotations++import json+import os+import sys++REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))+LCB_CHECKOUT = os.path.join(REPO_ROOT, "_local", "LiveCodeBench")+LCB_REPO = "livecodebench/code_generation_lite"+V6_SHARD = "test6.jsonl"+PROMPTS_PATH = os.path.join(REPO_ROOT, "datasets", "lcb_v6", "prompts.jsonl")+LCB_TESTS_PATH_ENV = "BITDECODING_LCB_TESTS_PATH"++if LCB_CHECKOUT not in sys.path:+ sys.path.insert(0, LCB_CHECKOUT)+++def _resolve_v6_path() -> str:+ """Use an explicit offline shard, or fall back to the Hub cache."""+ local_path = os.environ.get(LCB_TESTS_PATH_ENV)+ if local_path:+ path = os.path.abspath(os.path.expanduser(local_path))+ if not os.path.isfile(path):+ raise FileNotFoundError(+ f"{LCB_TESTS_PATH_ENV} points to a missing file: {path}"+ )+ return path++ from huggingface_hub import hf_hub_download++ return hf_hub_download(LCB_REPO, V6_SHARD, repo_type="dataset")+++def load_v6_problems():+ """The v6 contest window as ``CodeGenerationProblem``, in benchmark order."""+ from lcb_runner.benchmarks.code_generation import CodeGenerationProblem++ path = _resolve_v6_path()+ problems = []+ with open(path) as handle:+ for line in handle:+ line = line.strip()+ if line:+ problems.append(CodeGenerationProblem(**json.loads(line)))+ return problems+++def export_prompts(out_path: str = PROMPTS_PATH) -> int:+ """Write the fields generation needs, and only those.++ Test cases stay out of the file on purpose: the generating process must+ not be able to read the answers, and the private cases are most of the+ 134 MB.+ """+ problems = load_v6_problems()+ os.makedirs(os.path.dirname(out_path), exist_ok=True)+ with open(out_path, "w") as handle:+ for index, problem in enumerate(problems):+ handle.write(json.dumps({+ "bench_index": index,+ "question_id": problem.question_id,+ "question_title": problem.question_title,+ "question_content": problem.question_content,+ "starter_code": problem.starter_code,+ "platform": problem.platform.value,+ "difficulty": problem.difficulty.value,+ "contest_date": problem.contest_date.isoformat(),+ }, ensure_ascii=False) + "\n")+ return len(problems)+++def main() -> None:+ count = export_prompts()+ problems = load_v6_problems()+ dates = sorted(problem.contest_date for problem in problems)+ starters = sum(1 for problem in problems if problem.starter_code)+ print(f"[lcb] {count} problems -> {PROMPTS_PATH}")+ print(f"[lcb] contest dates {dates[0].date()} .. {dates[-1].date()}")+ print(f"[lcb] {starters} with starter code, {count - starters} stdin")+++if __name__ == "__main__":+ main()diff --git a/tools/prepare_data.py b/tools/prepare_data.pynew file mode 100755index 0000000..b434e1a--- /dev/null+++ b/tools/prepare_data.py@@ -0,0 +1,869 @@+#!/usr/bin/env python3+"""Stage every dataset BitDecoding-FP4 reads, from public sources only.++Nothing under ``datasets/`` is committed except ``MANIFEST.json``; this script+rebuilds the rest.++ python tools/prepare_data.py --steps eval+ python tools/prepare_data.py --steps long-context+ python tools/prepare_data.py --steps verify+"""+from __future__ import annotations++import argparse+import glob+import hashlib+import json+import os+import subprocess+import sys++REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))+sys.path.insert(0, REPO_ROOT)+DATASETS_ROOT = os.path.join(REPO_ROOT, "datasets")+MANIFEST_PATH = os.path.join(DATASETS_ROOT, "MANIFEST.json")+LONGBENCH_PRO_DIR = os.path.join(DATASETS_ROOT, "longbench_pro")+MRCR_DIR = os.path.join(DATASETS_ROOT, "mrcr")++LONGBENCH_PRO_REPO = "caskcsg/LongBench-Pro"+MRCR_REPO = "openai/mrcr"+AIME24_REPO = "HuggingFaceH4/aime_2024"+AIME25_REPO = "math-ai/aime25"+LONG_CONTEXT_MODEL_REPO = "Qwen/Qwen3-4B-Thinking-2507"+LONG_CONTEXT_REFERENCE_TOKENIZER_REPO = "Qwen/Qwen3-8B"+LONG_CONTEXT_MODEL_DIR = os.path.join(+ REPO_ROOT, "_local", "models", "Qwen3-4B-Thinking-2507"+)+LONG_CONTEXT_REFERENCE_TOKENIZER_DIR = os.path.join(+ REPO_ROOT, "_local", "models", "Qwen3-8B-tokenizer"+)+LONG_CONTEXT_MRCR_BUDGET = 229_120++RULER_SOURCE = os.environ.get(+ "BITDECODING_RULER_SOURCE", os.path.join(REPO_ROOT, "_local", "ruler_official")+)+RULER_REVISION = "c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a"+RULER_TASKS = (+ "niah_single_1", "niah_single_2", "niah_single_3",+ "niah_multikey_1", "niah_multikey_2", "niah_multikey_3",+ "niah_multivalue", "niah_multiquery",+)+RULER_MAX_SEQ_LENGTH = 122_880+RULER_NUM_SAMPLES = 500+RULER_SEED = 42+RULER_NUM_SHARDS = 8+RULER_SMOKE_LIMIT = 20++NEEDLE_GROUPS = (2, 4, 8)+STEPS = (+ "long-context-model", "longbench-pro", "mrcr", "long-context-mrcr",+ "aime24", "aime25", "gpqa", "lcb", "ruler", "verify",+ "long-context-verify",+)+EVAL_STEPS = (+ "longbench-pro", "mrcr", "aime24", "aime25", "lcb", "ruler", "gpqa",+ "verify",+)+LONG_CONTEXT_STEPS = (+ "long-context-model", "longbench-pro", "long-context-mrcr",+ "aime24", "aime25", "lcb", "gpqa", "ruler",+ "long-context-verify",+)+++# --------------------------------------------------------------------------- #+# hashing+# --------------------------------------------------------------------------- #+def _norm(obj):+ """Canonicalise a parquet cell so the digest ignores container types."""+ import numpy as np++ if isinstance(obj, np.ndarray):+ return [_norm(x) for x in obj.tolist()]+ if isinstance(obj, dict):+ return {k: _norm(v) for k, v in sorted(obj.items())}+ if isinstance(obj, (list, tuple)):+ return [_norm(x) for x in obj]+ if isinstance(obj, np.integer):+ return int(obj)+ if isinstance(obj, np.floating):+ return None if np.isnan(obj) else float(obj)+ if obj is not None and not isinstance(obj, (str, int, float, bool)):+ return str(obj)+ return obj+++def content_hash_rows(rows) -> str:+ """Order-sensitive digest over row dicts (writer/version independent)."""+ h = hashlib.sha256()+ for row in rows:+ h.update(json.dumps(_norm(row), ensure_ascii=False, sort_keys=True).encode())+ h.update(b"\n")+ return h.hexdigest()+++def content_hash_parquet(path: str) -> str:+ import pandas as pd++ df = pd.read_parquet(path)+ return content_hash_rows(df.to_dict(orient="records"))+++def sha256_file(path: str) -> str:+ h = hashlib.sha256()+ with open(path, "rb") as f:+ for chunk in iter(lambda: f.read(1 << 20), b""):+ h.update(chunk)+ return h.hexdigest()+++def load_manifest() -> dict:+ with open(MANIFEST_PATH) as f:+ return json.load(f)+++def _expected(manifest: dict, rel_path: str) -> dict:+ return manifest["derived"].get(rel_path, {})+++# --------------------------------------------------------------------------- #+# public downloads+# --------------------------------------------------------------------------- #+def step_longbench_pro(force: bool = False) -> None:+ from datasets import load_dataset++ target = LONGBENCH_PRO_DIR+ if os.path.isfile(os.path.join(target, "dataset_info.json")) and not force:+ print(f"[longbench-pro] present, skipping ({target})")+ return+ print(f"[longbench-pro] downloading {LONGBENCH_PRO_REPO} -> {target}")+ # Upstream ships one split and has renamed it: it was ``train`` when this+ # was written and is ``test`` now. Both are the same single split, so the+ # name is taken from the download rather than asserted.+ splits = load_dataset(+ LONGBENCH_PRO_REPO,+ revision=load_manifest()["upstream"][LONGBENCH_PRO_REPO]["revision"],+ )+ if len(splits) != 1:+ raise SystemExit(+ f"[longbench-pro] expected one split, got {sorted(splits)}"+ )+ (name,) = list(splits)+ print(f"[longbench-pro] split {name!r}, {len(splits[name])} rows")+ splits[name].save_to_disk(target)+++def _tokenizer_vocab_sha256(tokenizer) -> str:+ payload = json.dumps(+ sorted(tokenizer.get_vocab().items()), ensure_ascii=False,+ separators=(",", ":"),+ )+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()+++def verify_long_context_tokenizer(model_path: str = LONG_CONTEXT_MODEL_DIR) -> str:+ """Assert that the new checkpoint keeps the Qwen3-8B MRCR vocabulary."""+ from transformers import AutoTokenizer++ old = AutoTokenizer.from_pretrained(+ LONG_CONTEXT_REFERENCE_TOKENIZER_DIR, trust_remote_code=True,+ local_files_only=True,+ )+ new = AutoTokenizer.from_pretrained(+ model_path, trust_remote_code=True, local_files_only=True,+ )+ old_hash = _tokenizer_vocab_sha256(old)+ new_hash = _tokenizer_vocab_sha256(new)+ if old_hash != new_hash:+ raise SystemExit(+ "[long-context-tokenizer] Qwen3-4B-Thinking-2507 vocabulary differs "+ f"from Qwen3-8B: {new_hash} != {old_hash}"+ )+ print(f"[long-context-tokenizer] vocabulary match: {new_hash}")+ return new_hash+++def step_long_context_model(force: bool = False) -> None:+ from huggingface_hub import snapshot_download+ from transformers import AutoTokenizer++ config_path = os.path.join(LONG_CONTEXT_MODEL_DIR, "config.json")+ reference_config = os.path.join(+ LONG_CONTEXT_REFERENCE_TOKENIZER_DIR, "tokenizer_config.json"+ )+ if not os.path.isfile(reference_config):+ try:+ reference = AutoTokenizer.from_pretrained(+ LONG_CONTEXT_REFERENCE_TOKENIZER_REPO, local_files_only=True,+ trust_remote_code=True,+ )+ except OSError:+ reference = AutoTokenizer.from_pretrained(+ LONG_CONTEXT_REFERENCE_TOKENIZER_REPO, trust_remote_code=True,+ )+ os.makedirs(LONG_CONTEXT_REFERENCE_TOKENIZER_DIR, exist_ok=True)+ reference.save_pretrained(LONG_CONTEXT_REFERENCE_TOKENIZER_DIR)+ if os.path.isfile(config_path) and not force:+ print(f"[long-context-model] present, skipping ({LONG_CONTEXT_MODEL_DIR})")+ else:+ print(f"[long-context-model] downloading {LONG_CONTEXT_MODEL_REPO} -> {LONG_CONTEXT_MODEL_DIR}")+ snapshot_download(+ LONG_CONTEXT_MODEL_REPO, local_dir=LONG_CONTEXT_MODEL_DIR,+ revision=load_manifest()["upstream"][LONG_CONTEXT_MODEL_REPO]["revision"],+ )+ verify_long_context_tokenizer()+++def step_long_context_mrcr(force: bool = False, budget: int = LONG_CONTEXT_MRCR_BUDGET) -> None:+ from huggingface_hub import snapshot_download++ target = MRCR_DIR+ snapshot_download(+ MRCR_REPO,+ repo_type="dataset",+ local_dir=target,+ allow_patterns=[f"{n}needle/*" for n in NEEDLE_GROUPS],+ )+ filename = f"_filtered_le{budget}.jsonl"+ rel = f"datasets/mrcr/{filename}"+ build_mrcr_cache(+ LONG_CONTEXT_MODEL_DIR,+ force=force,+ budget=budget,+ out_path=os.path.join(target, filename),+ manifest_key=rel,+ )+++def step_mrcr(tokenizer: str, force: bool = False) -> None:+ from huggingface_hub import snapshot_download++ target = MRCR_DIR+ print(f"[mrcr] downloading {MRCR_REPO} shards -> {target} (~1.4 GB)")+ snapshot_download(+ MRCR_REPO,+ repo_type="dataset",+ local_dir=target,+ allow_patterns=[f"{n}needle/*" for n in NEEDLE_GROUPS],+ )+ build_mrcr_cache(tokenizer, force=force)+ build_mrcr_cache(+ tokenizer, force=force, budget=98048,+ out_path=os.path.join(MRCR_DIR, "_filtered_le98048.jsonl"),+ manifest_key="datasets/mrcr/_filtered_le98048.jsonl",+ )+++def build_mrcr_cache(+ tokenizer: str,+ force: bool = False,+ *,+ budget: int | None = None,+ out_path: str | None = None,+ manifest_key: str | None = None,+) -> None:+ """Keep MRCR rows whose prompt is <= max_prompt_tokens Qwen3 tokens."""+ import pandas as pd+ from transformers import AutoTokenizer++ manifest = load_manifest()+ manifest_key = (+ manifest_key or "datasets/mrcr/_filtered_le34000.jsonl"+ )+ spec = _expected(manifest, manifest_key)+ budget = budget if budget is not None else spec.get("max_prompt_tokens", 34000)+ out_path = out_path or os.path.join(MRCR_DIR, "_filtered_le34000.jsonl")+ if os.path.isfile(out_path) and not force:+ print(f"[mrcr-cache] present, skipping ({out_path})")+ return++ tok = AutoTokenizer.from_pretrained(tokenizer)+ rows = []+ for n in NEEDLE_GROUPS:+ shards = sorted(glob.glob(os.path.join(MRCR_DIR, f"{n}needle", "*.parquet")))+ if not shards:+ raise SystemExit(f"[mrcr-cache] no shards for {n}needle — run --steps mrcr first")+ for shard in shards:+ df = pd.read_parquet(shard)+ for _, r in df.iterrows():+ messages = json.loads(r["prompt"])+ encoded = tok([m["content"] for m in messages], add_special_tokens=False)+ n_tokens = sum(len(ids) for ids in encoded["input_ids"])+ if n_tokens > budget:+ continue+ rows.append({+ "id": None,+ "n_needles": n,+ "messages": messages,+ "answer": r["answer"],+ "random_string_to_prepend": r["random_string_to_prepend"],+ "prompt_tokens": n_tokens,+ })+ print(f"[mrcr-cache] {n}needle: kept {sum(1 for x in rows if x['n_needles'] == n)}")++ os.makedirs(os.path.dirname(out_path), exist_ok=True)+ with open(out_path, "w") as f:+ for row in rows:+ f.write(json.dumps(row, ensure_ascii=False) + "\n")+ print(f"[mrcr-cache] wrote {len(rows)} rows -> {out_path}")+ _report("mrcr cache", sha256_file(out_path), spec.get("sha256"))+ expected_rows = spec.get("n_rows")+ if expected_rows is not None and len(rows) != expected_rows:+ raise SystemExit(+ f"[mrcr-cache] row-count mismatch: {len(rows)} != {expected_rows}"+ )+++def step_aime25(force: bool = False) -> None:+ """AIME 2025, 30 problems, one row each.++ The 32 repeats Avg@32 needs are made at load time by ``evaluation.aime25`` rather+ than written out, so this file stays the upstream set and cannot drift from+ the constant that decides how many there are.+ """+ from datasets import load_dataset+ import pandas as pd++ out_path = os.path.join(DATASETS_ROOT, "aime25", "aime25.parquet")+ if os.path.isfile(out_path) and not force:+ print(f"[aime25] present, skipping ({out_path})")+ return+ splits = load_dataset(+ AIME25_REPO, revision=load_manifest()["upstream"][AIME25_REPO]["revision"],+ )+ if len(splits) != 1:+ raise SystemExit(f"[aime25] expected one split, got {sorted(splits)}")+ (name,) = list(splits)+ frame = splits[name].to_pandas()[["id", "problem", "answer"]]+ if len(frame) != 30:+ raise SystemExit(f"[aime25] expected 30 problems, got {len(frame)}")+ non_integer = [a for a in frame["answer"] if not str(a).strip().isdigit()]+ if non_integer:+ raise SystemExit(f"[aime25] non-integer answers: {non_integer[:5]}")+ os.makedirs(os.path.dirname(out_path), exist_ok=True)+ frame.to_parquet(out_path, index=False)+ print(f"[aime25] wrote {len(frame)} problems -> {out_path}")++++def step_aime24(force: bool = False) -> None:+ """AIME 2024, 30 problems. Avg@32 repeats are made at load time."""+ from datasets import load_dataset++ out_path = os.path.join(DATASETS_ROOT, "aime24", "aime24.parquet")+ if os.path.isfile(out_path) and not force:+ print(f"[aime24] present, skipping ({out_path})")+ return+ splits = load_dataset(+ AIME24_REPO, revision=load_manifest()["upstream"][AIME24_REPO]["revision"],+ )+ if len(splits) != 1:+ raise SystemExit(f"[aime24] expected one split, got {sorted(splits)}")+ (name,) = list(splits)+ frame = splits[name].to_pandas()+ if not {"problem", "answer"}.issubset(frame.columns):+ raise SystemExit(f"[aime24] missing problem/answer columns: {list(frame.columns)}")+ frame = frame[["problem", "answer"]]+ if len(frame) != 30:+ raise SystemExit(f"[aime24] expected 30 problems, got {len(frame)}")+ os.makedirs(os.path.dirname(out_path), exist_ok=True)+ frame.to_parquet(out_path, index=False)+ print(f"[aime24] wrote {len(frame)} problems -> {out_path}")+++def step_gpqa(force: bool = False) -> None:+ """Stage the 198-question GPQA Diamond CSV used by simple-evals."""+ import urllib.request+ import pandas as pd++ out_path = os.path.join(DATASETS_ROOT, "gpqa", "gpqa_diamond.parquet")+ if os.path.isfile(out_path) and not force:+ print(f"[gpqa] present, skipping ({out_path})")+ return+ url = "https://raw.githubusercontent.com/openai/simple-evals/main/data/gpqa_diamond.csv"+ os.makedirs(os.path.dirname(out_path), exist_ok=True)+ csv_path = out_path.removesuffix(".parquet") + ".csv"+ urllib.request.urlretrieve(url, csv_path)+ frame = pd.read_csv(csv_path)+ if len(frame) != 198:+ raise SystemExit(f"[gpqa] expected 198 Diamond rows, got {len(frame)}")+ frame.to_parquet(out_path, index=False)+ os.remove(csv_path)+ print(f"[gpqa] wrote {len(frame)} questions -> {out_path}")+++def step_lcb(force: bool = False) -> None:+ """Stage the v6 prompts and the official grader checkout."""+ from tools.lcb_data import LCB_CHECKOUT, PROMPTS_PATH, export_prompts++ if not os.path.isdir(os.path.join(LCB_CHECKOUT, "lcb_runner")):+ os.makedirs(os.path.dirname(LCB_CHECKOUT), exist_ok=True)+ subprocess.run([+ "git", "clone", "https://github.com/LiveCodeBench/LiveCodeBench.git",+ LCB_CHECKOUT,+ ], check=True)+ if os.path.isfile(PROMPTS_PATH) and not force:+ print(f"[lcb] present, skipping ({PROMPTS_PATH})")+ return+ count = export_prompts()+ if count != 175:+ raise SystemExit(f"[lcb] expected 175 v6 problems, got {count}")+ print(f"[lcb] wrote {count} prompts -> {PROMPTS_PATH}")+++def step_ruler(force: bool = False, tokenizer_path: str = LONG_CONTEXT_MODEL_REPO) -> None:+ """Generate the eight official RULER NIAH tasks at 122,880 tokens."""+ prepare = os.path.join(RULER_SOURCE, "scripts", "data", "prepare.py")+ revision_path = os.path.join(RULER_SOURCE, "REVISION")+ if not os.path.exists(RULER_SOURCE):+ os.makedirs(os.path.dirname(RULER_SOURCE), exist_ok=True)+ subprocess.run([+ "git", "clone", "https://github.com/NVIDIA/RULER.git", RULER_SOURCE,+ ], check=True)+ subprocess.run([+ "git", "-C", RULER_SOURCE, "checkout", "--detach", RULER_REVISION,+ ], check=True)+ if os.path.exists(os.path.join(RULER_SOURCE, ".git")):+ revision = subprocess.check_output(+ ["git", "-C", RULER_SOURCE, "rev-parse", "HEAD"], text=True,+ ).strip()+ else:+ with open(revision_path) as handle:+ revision = handle.read().strip()+ if not os.path.isfile(prepare):+ raise SystemExit(f"[ruler] missing generator: {prepare}")+ if revision != RULER_REVISION:+ raise SystemExit(+ f"[ruler] source revision {revision!r} != pinned {RULER_REVISION}"+ )++ out_dir = os.path.join(DATASETS_ROOT, "ruler")+ out_path = os.path.join(out_dir, "niah_122880.jsonl")+ provenance_path = os.path.join(out_dir, "generator.json")+ if os.path.isfile(out_path) and os.path.isfile(provenance_path) and not force:+ print(f"[ruler] present, skipping ({out_path})")+ return++ raw_dir = os.path.join(out_dir, "raw_122880")+ os.makedirs(raw_dir, exist_ok=True)+ env = dict(os.environ)+ env["PATH"] = os.path.dirname(sys.executable) + os.pathsep + env.get("PATH", "")+ for task in RULER_TASKS:+ command = [+ sys.executable,+ prepare,+ "--save_dir", raw_dir,+ "--benchmark", "synthetic",+ "--task", task,+ "--tokenizer_path", tokenizer_path,+ "--tokenizer_type", "hf",+ "--max_seq_length", str(RULER_MAX_SEQ_LENGTH),+ "--num_samples", str(RULER_NUM_SAMPLES),+ "--random_seed", str(RULER_SEED),+ "--model_template_type", "base",+ ]+ print(f"[ruler] generating {task}")+ subprocess.run(command, check=True, env=env)+ task_path = os.path.join(raw_dir, task, "validation.jsonl")+ if not os.path.isfile(task_path):+ raise SystemExit(f"[ruler] generator did not create {task_path}")+ with open(task_path) as handle:+ count = sum(bool(line.strip()) for line in handle)+ if count != RULER_NUM_SAMPLES:+ raise SystemExit(+ f"[ruler] {task} generated {count}, expected {RULER_NUM_SAMPLES}"+ )++ temporary = f"{out_path}.tmp.{os.getpid()}"+ shard_paths = [+ os.path.join(+ out_dir,+ f"niah_122880.shard_{index}_of_{RULER_NUM_SHARDS}.jsonl",+ )+ for index in range(RULER_NUM_SHARDS)+ ]+ shard_temporaries = [f"{path}.tmp.{os.getpid()}" for path in shard_paths]+ shard_handles = [open(path, "w") for path in shard_temporaries]+ smoke_paths = [+ os.path.join(+ out_dir,+ f"niah_122880.smoke_{RULER_SMOKE_LIMIT}.shard_{index}_of_"+ f"{RULER_NUM_SHARDS}.jsonl",+ )+ for index in range(RULER_NUM_SHARDS)+ ]+ smoke_temporaries = [f"{path}.tmp.{os.getpid()}" for path in smoke_paths]+ smoke_handles = [open(path, "w") for path in smoke_temporaries]+ smoke_base, smoke_remainder = divmod(RULER_SMOKE_LIMIT, len(RULER_TASKS))+ smoke_source_indices = {}+ for position, task in enumerate(RULER_TASKS):+ take = smoke_base + (position < smoke_remainder)+ if take == 1:+ selected = {0}+ else:+ selected = {+ index * (RULER_NUM_SAMPLES - 1) // (take - 1)+ for index in range(take)+ }+ smoke_source_indices[task] = selected+ bench_index = 0+ smoke_index = 0+ source_files = {}+ try:+ with open(temporary, "w") as output:+ for task in RULER_TASKS:+ task_path = os.path.join(raw_dir, task, "validation.jsonl")+ source_files[task] = {+ "path": os.path.relpath(task_path, REPO_ROOT),+ "sha256": sha256_file(task_path),+ "rows": RULER_NUM_SAMPLES,+ }+ with open(task_path) as source:+ for source_index, line in enumerate(source):+ if not line.strip():+ continue+ row = json.loads(line)+ row.update({+ "bench_index": bench_index,+ "task": task,+ "source_index": source_index,+ "generator_revision": revision,+ })+ rendered = json.dumps(row, ensure_ascii=False) + "\n"+ output.write(rendered)+ shard_handles[bench_index % RULER_NUM_SHARDS].write(rendered)+ if source_index in smoke_source_indices[task]:+ smoke_handles[+ smoke_index % RULER_NUM_SHARDS+ ].write(rendered)+ smoke_index += 1+ bench_index += 1+ output.flush()+ os.fsync(output.fileno())+ for handle in shard_handles:+ handle.flush()+ os.fsync(handle.fileno())+ for handle in smoke_handles:+ handle.flush()+ os.fsync(handle.fileno())+ finally:+ for handle in shard_handles + smoke_handles:+ handle.close()+ os.replace(temporary, out_path)+ for temporary_path, shard_path in zip(+ shard_temporaries, shard_paths, strict=True+ ):+ os.replace(temporary_path, shard_path)+ for temporary_path, shard_path in zip(+ smoke_temporaries, smoke_paths, strict=True+ ):+ os.replace(temporary_path, shard_path)+ if bench_index != len(RULER_TASKS) * RULER_NUM_SAMPLES:+ raise SystemExit(f"[ruler] combined row count is {bench_index}")+ if smoke_index != RULER_SMOKE_LIMIT:+ raise SystemExit(f"[ruler] smoke row count is {smoke_index}")++ from transformers import AutoTokenizer++ tokenizer = AutoTokenizer.from_pretrained(+ tokenizer_path, trust_remote_code=True+ )+ provenance = {+ "generator": "NVIDIA/RULER scripts/data/prepare.py",+ "generator_revision": revision,+ "random_seed": RULER_SEED,+ "tokenizer_path": (os.path.realpath(tokenizer_path)+ if os.path.isdir(tokenizer_path) else tokenizer_path),+ "tokenizer_vocab_sha256": _tokenizer_vocab_sha256(tokenizer),+ "max_seq_length": RULER_MAX_SEQ_LENGTH,+ "num_samples_per_task": RULER_NUM_SAMPLES,+ "tasks": list(RULER_TASKS),+ "source_files": source_files,+ "evaluation_shards": [+ {+ "path": os.path.relpath(path, REPO_ROOT),+ "sha256": sha256_file(path),+ "rows": bench_index // RULER_NUM_SHARDS,+ }+ for path in shard_paths+ ],+ "smoke_shards": [+ {+ "path": os.path.relpath(path, REPO_ROOT),+ "sha256": sha256_file(path),+ "rows": (+ RULER_SMOKE_LIMIT + RULER_NUM_SHARDS - 1 - index+ ) // RULER_NUM_SHARDS,+ }+ for index, path in enumerate(smoke_paths)+ ],+ "combined": {+ "path": os.path.relpath(out_path, REPO_ROOT),+ "sha256": sha256_file(out_path),+ "rows": bench_index,+ },+ }+ os.makedirs(os.path.dirname(provenance_path), exist_ok=True)+ with open(provenance_path, "w") as handle:+ json.dump(provenance, handle, indent=2, sort_keys=True)+ handle.write("\n")+ print(f"[ruler] wrote {bench_index} prompts -> {out_path}")+++# --------------------------------------------------------------------------- #+# verification+# --------------------------------------------------------------------------- #+def _report(label: str, got: str, expected: str) -> bool:+ if not expected:+ print(f" {label}: {got} (no expectation recorded)")+ return True+ ok = got == expected+ print(f" {'OK ' if ok else 'FAIL'} {label}: {got}{'' if ok else f' != {expected}'}")+ return ok+++def _jsonl_identity(path: str) -> tuple[int, str]:+ """Count non-empty rows and hash one large JSONL in a single pass."""+ digest = hashlib.sha256()+ rows = 0+ with open(path, "rb") as handle:+ for line in handle:+ digest.update(line)+ rows += bool(line.strip())+ return rows, digest.hexdigest()+++def step_verify() -> bool:+ """Hash every staged evaluation input and validate its registered size."""+ manifest = load_manifest()+ print("[verify] checking staged artifacts against datasets/MANIFEST.json")+ ok, missing = True, 0++ if os.path.isfile(os.path.join(LONGBENCH_PRO_DIR, "dataset_info.json")):+ from datasets import load_from_disk++ dataset = load_from_disk(LONGBENCH_PRO_DIR)+ if hasattr(dataset, "values"):+ values = list(dataset.values())+ if len(values) != 1:+ raise SystemExit(+ f"[verify] LongBench-Pro has {len(values)} splits, expected one"+ )+ dataset = values[0]+ spec = _expected(manifest, "datasets/longbench_pro")+ ok &= _report(+ "LongBench-Pro rows", str(len(dataset)), str(spec.get("n_rows", ""))+ )+ ok &= _report(+ "LongBench-Pro content sha256",+ content_hash_rows(dataset),+ spec.get("content_sha256"),+ )+ else:+ print(" MISS datasets/longbench_pro")+ missing += 1++ for rel, path in [+ ("datasets/mrcr/_filtered_le34000.jsonl",+ os.path.join(MRCR_DIR, "_filtered_le34000.jsonl")),+ ("datasets/mrcr/_filtered_le98048.jsonl",+ os.path.join(MRCR_DIR, "_filtered_le98048.jsonl")),+ ("datasets/aime24/aime24.parquet",+ os.path.join(DATASETS_ROOT, "aime24", "aime24.parquet")),+ ("datasets/aime25/aime25.parquet",+ os.path.join(DATASETS_ROOT, "aime25", "aime25.parquet")),+ ("datasets/gpqa/gpqa_diamond.parquet",+ os.path.join(DATASETS_ROOT, "gpqa", "gpqa_diamond.parquet")),+ ("datasets/lcb_v6/prompts.jsonl",+ os.path.join(DATASETS_ROOT, "lcb_v6", "prompts.jsonl")),+ ("datasets/ruler/niah_122880.jsonl",+ os.path.join(DATASETS_ROOT, "ruler", "niah_122880.jsonl")),+ ]:+ spec = _expected(manifest, rel)+ if not os.path.exists(path):+ print(f" MISS {rel}")+ missing += 1+ continue+ if path.endswith(".parquet"):+ import pandas as pd++ rows = len(pd.read_parquet(path))+ ok &= _report(f"{rel} rows", str(rows), str(spec.get("n_rows", "")))+ if spec.get("content_sha256"):+ ok &= _report(+ f"{rel} content sha256",+ content_hash_parquet(path),+ spec["content_sha256"],+ )+ else:+ ok &= _report(f"{rel} sha256", sha256_file(path), spec.get("sha256"))+ else:+ rows, digest = _jsonl_identity(path)+ ok &= _report(f"{rel} rows", str(rows), str(spec.get("n_rows", "")))+ ok &= _report(f"{rel} sha256", digest, spec.get("sha256"))++ ruler_provenance = os.path.join(DATASETS_ROOT, "ruler", "generator.json")+ if not os.path.isfile(ruler_provenance):+ print(" MISS datasets/ruler/generator.json")+ missing += 1+ else:+ spec = _expected(manifest, "datasets/ruler/generator.json")+ ok &= _report(+ "datasets/ruler/generator.json sha256",+ sha256_file(ruler_provenance),+ spec.get("sha256"),+ )+ with open(ruler_provenance) as handle:+ provenance = json.load(handle)+ for identity in (+ provenance.get("evaluation_shards", [])+ + provenance.get("smoke_shards", [])+ ):+ path = os.path.join(REPO_ROOT, identity["path"])+ if not os.path.isfile(path):+ print(f" MISS {identity['path']}")+ missing += 1+ continue+ rows, digest = _jsonl_identity(path)+ ok &= _report(+ f"{identity['path']} rows", str(rows), str(identity.get("rows", ""))+ )+ ok &= _report(+ f"{identity['path']} sha256", digest, identity.get("sha256")+ )+ if missing:+ print(f"[verify] {missing} artifact(s) not staged — run --steps eval")+ return ok and missing == 0+++def step_long_context_verify() -> bool:+ """Verify the long-context model, tokenizer, and MRCR cache."""+ config_path = os.path.join(LONG_CONTEXT_MODEL_DIR, "config.json")+ index_path = os.path.join(+ LONG_CONTEXT_MODEL_DIR, "model.safetensors.index.json"+ )+ cache_path = os.path.join(MRCR_DIR, "_filtered_le229120.jsonl")+ if not os.path.isfile(config_path):+ print(f" MISS {config_path}")+ return False+ with open(config_path) as handle:+ model_config = json.load(handle)+ native_context = int(model_config.get("max_position_embeddings", 0))+ if native_context < 262_144:+ print(f" FAIL native context: {native_context} < 262144")+ return False+ manifest = load_manifest()+ model_spec = manifest["upstream"].get(LONG_CONTEXT_MODEL_REPO, {})+ ok = True+ ok &= _report(+ "long-context model config sha256",+ sha256_file(config_path),+ model_spec.get("config_sha256"),+ )+ if not os.path.isfile(index_path):+ print(f" MISS {index_path}")+ return False+ ok &= _report(+ "long-context model index sha256",+ sha256_file(index_path),+ model_spec.get("model_index_sha256"),+ )+ vocab_hash = verify_long_context_tokenizer()+ ok &= _report(+ "long-context tokenizer vocabulary sha256",+ vocab_hash,+ model_spec.get("tokenizer_vocab_sha256"),+ )+ if not os.path.isfile(cache_path):+ print(f" MISS {cache_path}")+ return False+ count = 0+ maximum = 0+ with open(cache_path) as handle:+ for line in handle:+ if not line.strip():+ continue+ row = json.loads(line)+ count += 1+ maximum = max(maximum, int(row["prompt_tokens"]))+ spec = _expected(load_manifest(), "datasets/mrcr/_filtered_le229120.jsonl")+ expected_rows = spec.get("n_rows")+ ok &= expected_rows in (None, count) and maximum <= LONG_CONTEXT_MRCR_BUDGET+ print(+ f" {'OK ' if ok else 'FAIL'} long-context MRCR: "+ f"rows={count}, max_prompt_tokens={maximum}"+ )+ expected_hash = spec.get("sha256")+ if expected_hash:+ ok &= _report("long-context MRCR sha256", sha256_file(cache_path), expected_hash)+ return ok+++def main() -> int:+ default_tokenizer = os.environ.get("BITDECODING_TOKENIZER") or os.environ.get(+ "MODEL_PATH") or LONG_CONTEXT_MODEL_REPO+ parser = argparse.ArgumentParser(+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter+ )+ parser.add_argument(+ "--steps", nargs="+", default=["eval"],+ choices=list(STEPS) + ["all", "eval", "long-context"],+ help="'eval' = all seven paper datasets; "+ "'long-context' = native-262k checkpoint and cache",+ )+ parser.add_argument("--tokenizer", default=default_tokenizer,+ help="Qwen3 tokenizer for the MRCR length filter")+ parser.add_argument(+ "--long-context-mrcr-budget", type=int, default=LONG_CONTEXT_MRCR_BUDGET,+ help="long-context evaluation MRCR prompt cap (the 131k fallback uses 98048)",+ )+ parser.add_argument("--force", action="store_true",+ help="rebuild artifacts that already exist")+ args = parser.parse_args()++ aliases = {+ "all": list(STEPS),+ "eval": list(EVAL_STEPS),+ "long-context": list(LONG_CONTEXT_STEPS),+ }+ steps = []+ for s in args.steps:+ steps.extend(aliases.get(s, [s]))+ seen, ordered = set(), []+ for s in STEPS:+ if s in steps and s not in seen:+ seen.add(s)+ ordered.append(s)++ for step in ordered:+ if step == "long-context-model":+ step_long_context_model(args.force)+ elif step == "longbench-pro":+ step_longbench_pro(args.force)+ elif step == "mrcr":+ step_mrcr(args.tokenizer, args.force)+ elif step == "long-context-mrcr":+ step_long_context_mrcr(args.force, args.long_context_mrcr_budget)+ elif step == "aime24":+ step_aime24(args.force)+ elif step == "aime25":+ step_aime25(args.force)+ elif step == "gpqa":+ step_gpqa(args.force)+ elif step == "lcb":+ step_lcb(args.force)+ elif step == "ruler":+ step_ruler(args.force, args.tokenizer)+ elif step == "verify":+ if not step_verify():+ print("\n[verify] MISMATCH — staged data differs from MANIFEST.json.")+ return 1+ elif step == "long-context-verify":+ if not step_long_context_verify():+ print("\n[long-context-verify] MISMATCH")+ return 1+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/score_lcb.py b/tools/score_lcb.pynew file mode 100644index 0000000..75bf581--- /dev/null+++ b/tools/score_lcb.py@@ -0,0 +1,182 @@+#!/usr/bin/env python3+"""Grade LiveCodeBench generations with the official checker, in a sandbox.++Runs in a **new process** of the inference environment+(``_local/envs/bd_fp4``), never in a process that already holds a CUDA+context. LiveCodeBench is imported from a checkout on ``PYTHONPATH``; it is+not pip-installed, because that would replace this repository's vLLM.++Grading itself is ``lcb_runner.evaluation.codegen_metrics`` -- the same+function ``custom_evaluator`` calls. What is skipped is that CLI's benchmark+load, because it goes through a loader that downloads 4,486 MB to read the 134+that hold v6 (see tools/lcb_data.py), and its alignment step, which sorts+generations by ``str(question_id)`` and zips them against the benchmark.+Records carry ``bench_index`` instead, so alignment is positional and cannot+silently pair a solution with another problem's tests.++Isolation is a user namespace with no network and its own PID namespace, plus+an address-space limit. LCB's ``reliability_guard`` says in its own docstring+that it is "not a security sandbox" -- it nulls out some Python attributes --+and it is called with ``maximum_memory_bytes=None``, so nothing else here caps+memory.++ _local/envs/bd_fp4/bin/python tools/score_lcb.py \\+ --run-dir _local/eval_smoke --mode bitdecoding-nvfp4+"""+from __future__ import annotations++import argparse+import glob+import json+import os+import resource+import sys++REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))+sys.path.insert(0, REPO_ROOT)++from tools.lcb_data import load_v6_problems # noqa: E402+from evaluation.lcb import REPEATS # noqa: E402++SANDBOX_MARKER = "BITDECODING_LCB_SANDBOXED"+++def _reexec_sandboxed(memory_gib: int) -> None:+ """Re-run this program inside a network-less user + PID namespace."""+ os.environ[SANDBOX_MARKER] = "1"+ os.environ["BITDECODING_LCB_MEMORY_GIB"] = str(memory_gib)+ command = [+ "unshare", "--user", "--map-root-user", "--net", "--pid", "--fork",+ sys.executable, os.path.abspath(__file__), *sys.argv[1:],+ ]+ os.execvp(command[0], command)+++def load_records(run_dir: str, mode: str) -> dict:+ """Every graded row for one arm, keyed by its position in the benchmark."""+ pattern = os.path.join(run_dir, "**", f"lcb__{mode}.jsonl")+ paths = sorted(glob.glob(pattern, recursive=True))+ if not paths:+ raise SystemExit(f"no lcb__{mode}.jsonl under {run_dir}")+ by_index, source = {}, {}+ for path in paths:+ with open(path) as handle:+ for line in handle:+ line = line.strip()+ if not line:+ continue+ record = json.loads(line)+ index = (+ record.get("problem_index", record["bench_index"]),+ record.get("repeat", 0),+ )+ if index in by_index:+ raise SystemExit(+ f"row {index} appears in both "+ f"{source[index]} and {path}"+ )+ by_index[index] = record+ source[index] = path+ return by_index, source+++def write_back(source: dict, by_index: dict) -> None:+ """Rewrite each shard file with ``pass1`` filled in, order preserved."""+ for path in sorted(set(source.values())):+ with open(path) as handle:+ rows = [json.loads(line) for line in handle if line.strip()]+ for row in rows:+ key = (+ row.get("problem_index", row["bench_index"]),+ row.get("repeat", 0),+ )+ row["pass1"] = by_index[key]["pass1"]+ with open(path, "w") as handle:+ for row in rows:+ handle.write(json.dumps(row, ensure_ascii=False) + "\n")+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--run-dir", required=True)+ parser.add_argument("--mode", required=True)+ parser.add_argument("--num-process", type=int, default=32)+ parser.add_argument("--timeout", type=int, default=6)+ parser.add_argument("--memory-gib", type=int, default=16,+ help="address-space cap for this process and its children")+ parser.add_argument("--no-sandbox", action="store_true")+ parser.add_argument("--allow-partial", action="store_true",+ help="grade a subset (smoke runs), instead of requiring "+ "every problem in the slice")+ args = parser.parse_args()++ if not args.no_sandbox and not os.environ.get(SANDBOX_MARKER):+ _reexec_sandboxed(args.memory_gib)++ limit = int(os.environ.get("BITDECODING_LCB_MEMORY_GIB", args.memory_gib)) * 1024 ** 3+ resource.setrlimit(resource.RLIMIT_AS, (limit, limit))++ from lcb_runner.evaluation import codegen_metrics, extract_instance_results++ problems = load_v6_problems()+ by_index, source = load_records(args.run_dir, args.mode)+ keys = sorted(by_index)+ problem_indices = sorted({key[0] for key in keys})+ if problem_indices and (problem_indices[-1] >= len(problems)):+ raise SystemExit(+ f"problem_index {problem_indices[-1]} is outside the {len(problems)}-problem "+ "slice; the prompts and the grader disagree about the benchmark"+ )+ expected_keys = {+ (problem_index, repeat)+ for problem_index in problem_indices+ for repeat in range(REPEATS)+ }+ missing = expected_keys - set(keys)+ if missing and not args.allow_partial:+ raise SystemExit(f"missing repeat rows, for example {sorted(missing)[:3]}")+ if len(problem_indices) != len(problems) and not args.allow_partial:+ raise SystemExit(+ f"{len(problem_indices)} of {len(problems)} problems present; pass "+ "--allow-partial to grade a subset"+ )++ samples = [problems[key[0]].get_evaluation_sample() for key in keys]+ generations = [[by_index[key]["code"] or ""] for key in keys]+ empty = sum(1 for g in generations if not g[0].strip())+ print(+ f"[lcb] grading {len(keys)} rows for {args.mode} "+ f"({empty} with no extracted code)",+ flush=True,+ )++ metrics, results, _metadata = codegen_metrics(+ samples, generations,+ num_process_evaluate=args.num_process,+ timeout=args.timeout,+ )+ graded = extract_instance_results(results)++ for position, key in enumerate(keys):+ by_index[key]["pass1"] = float(bool(graded[position][0]))+ write_back(source, by_index)++ per_problem = {}+ for key in keys:+ per_problem.setdefault(key[0], []).append(by_index[key]["pass1"])+ if not args.allow_partial:+ bad = [index for index, values in per_problem.items() if len(values) != REPEATS]+ if bad:+ raise SystemExit(f"problem repeats incomplete, for example {bad[:3]}")+ problem_means = [sum(values) / len(values) for values in per_problem.values()]+ passed = sum(problem_means)+ print(+ f"[lcb] Avg@{REPEATS} = {passed / len(problem_means):.4f} "+ f"({passed:.2f}/{len(problem_means)} problems)"+ )+ print(f"[lcb] official metrics: {metrics}")+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())