Accurate, fast FP4 KV-cache decoding through algorithm-system co-design.
Simplify large-model results wording and table footnotes
DD-DuDafebe5a7f9f97
diff --git a/README.md b/README.mdindex c52a2fc..551fa3c 100644--- a/README.md+++ b/README.md@@ -15,9 +15,9 @@ Qwen3-4B-Thinking-2507 and Qwen3-30B-A3B-Thinking-2507.-Paper results on larger FP8-weight models:+Results on larger FP8-weight models:-+## Higher Serving Throughput.diff --git a/imgs/large-model-accuracy.svg b/imgs/large-model-accuracy.svgindex c755f02..c567b4b 100644--- a/imgs/large-model-accuracy.svg+++ b/imgs/large-model-accuracy.svg@@ -1,7 +1,7 @@<?xml version="1.0" encoding="utf-8" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN""http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">-<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="864pt" height="406.8pt" viewBox="0 0 864 406.8" xmlns="http://www.w3.org/2000/svg" version="1.1">+<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="864pt" height="385.2pt" viewBox="0 0 864 385.2" xmlns="http://www.w3.org/2000/svg" version="1.1"><desc>{"source": "Paper large-model accuracy table", "columns": ["Method", "LB-Pro", "MRCR", "AIME25", "KV (GiB)"], "weight_dtype": "FP8", "models": [{"model": "GLM-4.7", "parameters": "355B", "attention": "GQA", "context": "200K", "rows": [{"method": "BF16 KV", "values": ["67.40", "61.85", "93.23", "71.88"]}, {"method": "FP8 KV", "values": ["65.87", "61.27", "92.40", "35.94"]}, {"method": "TRT-LLM-NVFP4", "values": ["62.28", "56.29", "91.88", "20.21"]}, {"method": "BitDecoding-NVFP4", "values": ["66.74", "60.83", "93.75", "20.27"]}]}, {"model": "Hy4-preview", "parameters": "770B", "attention": "MLA", "context": "1M", "rows": [{"method": "BF16 KV", "values": ["60.29", "67.96", "98.65", "90.46"]}, {"method": "FP8 KV", "values": ["59.71", "67.83", "98.96", "52.68"]}, {"method": "BitDecoding-NVFP4", "values": ["60.32", "67.84", "98.75", "34.41"]}]}]}</desc><title>Large-model accuracy and KV-cache memory</title><metadata>@@ -24,8 +24,8 @@</defs><g id="figure_1"><g id="patch_1">- <path d="M 0 406.8-L 864 406.8+ <path d="M 0 385.2+L 864 385.2L 864 0L 0 0z@@ -38,7 +38,7 @@ L 835.2 174.96L 835.2 203.76L 28.8 203.76z-" clip-path="url(#pdee8638843)" style="fill: #edf4e9"/>+" clip-path="url(#p101463c74f)" style="fill: #edf4e9"/></g><g id="patch_3"><path d="M 28.8 306@@ -46,27 +46,27 @@ L 835.2 306L 835.2 334.8L 28.8 334.8z-" clip-path="url(#pdee8638843)" style="fill: #edf4e9"/>+" clip-path="url(#p101463c74f)" style="fill: #edf4e9"/></g><g id="line2d_1"><path d="M 28.8 14.4L 835.2 14.4-" clip-path="url(#pdee8638843)" style="fill: none; stroke: #222222; stroke-width: 1.4"/>+" clip-path="url(#p101463c74f)" style="fill: none; stroke: #222222; stroke-width: 1.4"/></g><g id="line2d_2"><path d="M 28.8 56.16L 835.2 56.16-" clip-path="url(#pdee8638843)" style="fill: none; stroke: #222222; stroke-width: 1.1"/>+" clip-path="url(#p101463c74f)" style="fill: none; stroke: #222222; stroke-width: 1.1"/></g><g id="line2d_3"><path d="M 28.8 210.24L 835.2 210.24-" clip-path="url(#pdee8638843)" style="fill: none; stroke: #555555"/>+" clip-path="url(#p101463c74f)" style="fill: none; stroke: #555555"/></g><g id="line2d_4"><path d="M 28.8 341.28L 835.2 341.28-" clip-path="url(#pdee8638843)" style="fill: none; stroke: #222222; stroke-width: 1.4"/>+" clip-path="url(#p101463c74f)" style="fill: none; stroke: #222222; stroke-width: 1.4"/></g><g id="text_1"><!-- Method -->@@ -2406,99 +2406,16 @@ z<use xlink:href="#DejaVuSans-11" transform="translate(3812.734375 0)"/></g></g>- <g id="text_46">- <!-- LB-Pro: LongBench Pro. AIME25: AIME 2025. -->- <g style="fill: #555555" transform="translate(28.8 394.888247) scale(0.1512 -0.1512)">- <defs>- <path id="DejaVuSans-1d" d="M 750 794-L 1409 794-L 1409 0-L 750 0-L 750 794-z-M 750 3309-L 1409 3309-L 1409 2516-L 750 2516-L 750 3309-z-" transform="scale(0.015625)"/>- <path id="DejaVuSans-2c" d="M 628 4666-L 1259 4666-L 1259 0-L 628 0-L 628 4666-z-" transform="scale(0.015625)"/>- <path id="DejaVuSans-28" d="M 628 4666-L 3578 4666-L 3578 4134-L 1259 4134-L 1259 2753-L 3481 2753-L 3481 2222-L 1259 2222-L 1259 531-L 3634 531-L 3634 0-L 628 0-L 628 4666-z-" transform="scale(0.015625)"/>- </defs>- <use xlink:href="#DejaVuSans-2f"/>- <use xlink:href="#DejaVuSans-25" transform="translate(55.71875 0)"/>- <use xlink:href="#DejaVuSans-10" transform="translate(124.328125 0)"/>- <use xlink:href="#DejaVuSans-33" transform="translate(160.40625 0)"/>- <use xlink:href="#DejaVuSans-55" transform="translate(218.953125 0)"/>- <use xlink:href="#DejaVuSans-52" transform="translate(257.859375 0)"/>- <use xlink:href="#DejaVuSans-1d" transform="translate(319.046875 0)"/>- <use xlink:href="#DejaVuSans-3" transform="translate(352.734375 0)"/>- <use xlink:href="#DejaVuSans-2f" transform="translate(384.515625 0)"/>- <use xlink:href="#DejaVuSans-52" transform="translate(438.484375 0)"/>- <use xlink:href="#DejaVuSans-51" transform="translate(499.671875 0)"/>- <use xlink:href="#DejaVuSans-4a" transform="translate(563.046875 0)"/>- <use xlink:href="#DejaVuSans-25" transform="translate(626.53125 0)"/>- <use xlink:href="#DejaVuSans-48" transform="translate(695.140625 0)"/>- <use xlink:href="#DejaVuSans-51" transform="translate(756.671875 0)"/>- <use xlink:href="#DejaVuSans-46" transform="translate(820.046875 0)"/>- <use xlink:href="#DejaVuSans-4b" transform="translate(875.03125 0)"/>- <use xlink:href="#DejaVuSans-3" transform="translate(938.40625 0)"/>- <use xlink:href="#DejaVuSans-33" transform="translate(970.1875 0)"/>- <use xlink:href="#DejaVuSans-55" transform="translate(1028.734375 0)"/>- <use xlink:href="#DejaVuSans-52" transform="translate(1067.640625 0)"/>- <use xlink:href="#DejaVuSans-11" transform="translate(1127.078125 0)"/>- <use xlink:href="#DejaVuSans-3" transform="translate(1158.859375 0)"/>- <use xlink:href="#DejaVuSans-24" transform="translate(1190.640625 0)"/>- <use xlink:href="#DejaVuSans-2c" transform="translate(1259.046875 0)"/>- <use xlink:href="#DejaVuSans-30" transform="translate(1288.546875 0)"/>- <use xlink:href="#DejaVuSans-28" transform="translate(1374.828125 0)"/>- <use xlink:href="#DejaVuSans-15" transform="translate(1438.015625 0)"/>- <use xlink:href="#DejaVuSans-18" transform="translate(1501.640625 0)"/>- <use xlink:href="#DejaVuSans-1d" transform="translate(1565.265625 0)"/>- <use xlink:href="#DejaVuSans-3" transform="translate(1598.953125 0)"/>- <use xlink:href="#DejaVuSans-24" transform="translate(1630.734375 0)"/>- <use xlink:href="#DejaVuSans-2c" transform="translate(1699.140625 0)"/>- <use xlink:href="#DejaVuSans-30" transform="translate(1728.640625 0)"/>- <use xlink:href="#DejaVuSans-28" transform="translate(1814.921875 0)"/>- <use xlink:href="#DejaVuSans-3" transform="translate(1878.109375 0)"/>- <use xlink:href="#DejaVuSans-15" transform="translate(1909.890625 0)"/>- <use xlink:href="#DejaVuSans-13" transform="translate(1973.515625 0)"/>- <use xlink:href="#DejaVuSans-15" transform="translate(2037.140625 0)"/>- <use xlink:href="#DejaVuSans-18" transform="translate(2100.765625 0)"/>- <use xlink:href="#DejaVuSans-11" transform="translate(2164.390625 0)"/>- </g>- </g><g id="line2d_5"><path d="M 691.2 14.4L 691.2 341.28-" clip-path="url(#pdee8638843)" style="fill: none; stroke: #aaaaaa; stroke-width: 0.65; stroke-linecap: square"/>+" clip-path="url(#p101463c74f)" style="fill: none; stroke: #aaaaaa; stroke-width: 0.65; stroke-linecap: square"/></g></g></g><defs>- <clipPath id="pdee8638843">- <rect x="0" y="0" width="864" height="406.8"/>+ <clipPath id="p101463c74f">+ <rect x="0" y="0" width="864" height="385.2"/></clipPath></defs></svg>