Skip to content

Accurate, fast FP4 KV-cache decoding through algorithm-system co-design.

Simplify large-model results wording and table footnotes

DD-DuDafebe5a7f9f97
diff --git a/README.md b/README.md
index c52a2fc..551fa3c 100644
--- a/README.md
+++ b/README.md
@@ -15,9 +15,9 @@ Qwen3-4B-Thinking-2507 and Qwen3-30B-A3B-Thinking-2507.
![Paper accuracy table: scores on seven benchmarks and their average for both Qwen3 models.](imgs/accuracy.svg)
-Paper results on larger FP8-weight models:
+Results on larger FP8-weight models:
-![Paper accuracy and KV-cache memory table for GLM-4.7 (GQA) and Hy4-preview (MLA).](imgs/large-model-accuracy.svg)
+![Accuracy and KV-cache memory table for GLM-4.7 (GQA) and Hy4-preview (MLA).](imgs/large-model-accuracy.svg)
## Higher Serving Throughput.
diff --git a/imgs/large-model-accuracy.svg b/imgs/large-model-accuracy.svg
index c755f02..c567b4b 100644
--- a/imgs/large-model-accuracy.svg
+++ b/imgs/large-model-accuracy.svg
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="864pt" height="406.8pt" viewBox="0 0 864 406.8" xmlns="http://www.w3.org/2000/svg" version="1.1">
+<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="864pt" height="385.2pt" viewBox="0 0 864 385.2" xmlns="http://www.w3.org/2000/svg" version="1.1">
<desc>{&quot;source&quot;: &quot;Paper large-model accuracy table&quot;, &quot;columns&quot;: [&quot;Method&quot;, &quot;LB-Pro&quot;, &quot;MRCR&quot;, &quot;AIME25&quot;, &quot;KV (GiB)&quot;], &quot;weight_dtype&quot;: &quot;FP8&quot;, &quot;models&quot;: [{&quot;model&quot;: &quot;GLM-4.7&quot;, &quot;parameters&quot;: &quot;355B&quot;, &quot;attention&quot;: &quot;GQA&quot;, &quot;context&quot;: &quot;200K&quot;, &quot;rows&quot;: [{&quot;method&quot;: &quot;BF16 KV&quot;, &quot;values&quot;: [&quot;67.40&quot;, &quot;61.85&quot;, &quot;93.23&quot;, &quot;71.88&quot;]}, {&quot;method&quot;: &quot;FP8 KV&quot;, &quot;values&quot;: [&quot;65.87&quot;, &quot;61.27&quot;, &quot;92.40&quot;, &quot;35.94&quot;]}, {&quot;method&quot;: &quot;TRT-LLM-NVFP4&quot;, &quot;values&quot;: [&quot;62.28&quot;, &quot;56.29&quot;, &quot;91.88&quot;, &quot;20.21&quot;]}, {&quot;method&quot;: &quot;BitDecoding-NVFP4&quot;, &quot;values&quot;: [&quot;66.74&quot;, &quot;60.83&quot;, &quot;93.75&quot;, &quot;20.27&quot;]}]}, {&quot;model&quot;: &quot;Hy4-preview&quot;, &quot;parameters&quot;: &quot;770B&quot;, &quot;attention&quot;: &quot;MLA&quot;, &quot;context&quot;: &quot;1M&quot;, &quot;rows&quot;: [{&quot;method&quot;: &quot;BF16 KV&quot;, &quot;values&quot;: [&quot;60.29&quot;, &quot;67.96&quot;, &quot;98.65&quot;, &quot;90.46&quot;]}, {&quot;method&quot;: &quot;FP8 KV&quot;, &quot;values&quot;: [&quot;59.71&quot;, &quot;67.83&quot;, &quot;98.96&quot;, &quot;52.68&quot;]}, {&quot;method&quot;: &quot;BitDecoding-NVFP4&quot;, &quot;values&quot;: [&quot;60.32&quot;, &quot;67.84&quot;, &quot;98.75&quot;, &quot;34.41&quot;]}]}]}</desc>
<title>Large-model accuracy and KV-cache memory</title>
<metadata>
@@ -24,8 +24,8 @@
</defs>
<g id="figure_1">
<g id="patch_1">
- <path d="M 0 406.8
-L 864 406.8
+ <path d="M 0 385.2
+L 864 385.2
L 864 0
L 0 0
z
@@ -38,7 +38,7 @@ L 835.2 174.96
L 835.2 203.76
L 28.8 203.76
z
-" clip-path="url(#pdee8638843)" style="fill: #edf4e9"/>
+" clip-path="url(#p101463c74f)" style="fill: #edf4e9"/>
</g>
<g id="patch_3">
<path d="M 28.8 306
@@ -46,27 +46,27 @@ L 835.2 306
L 835.2 334.8
L 28.8 334.8
z
-" clip-path="url(#pdee8638843)" style="fill: #edf4e9"/>
+" clip-path="url(#p101463c74f)" style="fill: #edf4e9"/>
</g>
<g id="line2d_1">
<path d="M 28.8 14.4
L 835.2 14.4
-" clip-path="url(#pdee8638843)" style="fill: none; stroke: #222222; stroke-width: 1.4"/>
+" clip-path="url(#p101463c74f)" style="fill: none; stroke: #222222; stroke-width: 1.4"/>
</g>
<g id="line2d_2">
<path d="M 28.8 56.16
L 835.2 56.16
-" clip-path="url(#pdee8638843)" style="fill: none; stroke: #222222; stroke-width: 1.1"/>
+" clip-path="url(#p101463c74f)" style="fill: none; stroke: #222222; stroke-width: 1.1"/>
</g>
<g id="line2d_3">
<path d="M 28.8 210.24
L 835.2 210.24
-" clip-path="url(#pdee8638843)" style="fill: none; stroke: #555555"/>
+" clip-path="url(#p101463c74f)" style="fill: none; stroke: #555555"/>
</g>
<g id="line2d_4">
<path d="M 28.8 341.28
L 835.2 341.28
-" clip-path="url(#pdee8638843)" style="fill: none; stroke: #222222; stroke-width: 1.4"/>
+" clip-path="url(#p101463c74f)" style="fill: none; stroke: #222222; stroke-width: 1.4"/>
</g>
<g id="text_1">
<!-- Method -->
@@ -2406,99 +2406,16 @@ z
<use xlink:href="#DejaVuSans-11" transform="translate(3812.734375 0)"/>
</g>
</g>
- <g id="text_46">
- <!-- LB-Pro: LongBench Pro. AIME25: AIME 2025. -->
- <g style="fill: #555555" transform="translate(28.8 394.888247) scale(0.1512 -0.1512)">
- <defs>
- <path id="DejaVuSans-1d" d="M 750 794
-L 1409 794
-L 1409 0
-L 750 0
-L 750 794
-z
-M 750 3309
-L 1409 3309
-L 1409 2516
-L 750 2516
-L 750 3309
-z
-" transform="scale(0.015625)"/>
- <path id="DejaVuSans-2c" d="M 628 4666
-L 1259 4666
-L 1259 0
-L 628 0
-L 628 4666
-z
-" transform="scale(0.015625)"/>
- <path id="DejaVuSans-28" d="M 628 4666
-L 3578 4666
-L 3578 4134
-L 1259 4134
-L 1259 2753
-L 3481 2753
-L 3481 2222
-L 1259 2222
-L 1259 531
-L 3634 531
-L 3634 0
-L 628 0
-L 628 4666
-z
-" transform="scale(0.015625)"/>
- </defs>
- <use xlink:href="#DejaVuSans-2f"/>
- <use xlink:href="#DejaVuSans-25" transform="translate(55.71875 0)"/>
- <use xlink:href="#DejaVuSans-10" transform="translate(124.328125 0)"/>
- <use xlink:href="#DejaVuSans-33" transform="translate(160.40625 0)"/>
- <use xlink:href="#DejaVuSans-55" transform="translate(218.953125 0)"/>
- <use xlink:href="#DejaVuSans-52" transform="translate(257.859375 0)"/>
- <use xlink:href="#DejaVuSans-1d" transform="translate(319.046875 0)"/>
- <use xlink:href="#DejaVuSans-3" transform="translate(352.734375 0)"/>
- <use xlink:href="#DejaVuSans-2f" transform="translate(384.515625 0)"/>
- <use xlink:href="#DejaVuSans-52" transform="translate(438.484375 0)"/>
- <use xlink:href="#DejaVuSans-51" transform="translate(499.671875 0)"/>
- <use xlink:href="#DejaVuSans-4a" transform="translate(563.046875 0)"/>
- <use xlink:href="#DejaVuSans-25" transform="translate(626.53125 0)"/>
- <use xlink:href="#DejaVuSans-48" transform="translate(695.140625 0)"/>
- <use xlink:href="#DejaVuSans-51" transform="translate(756.671875 0)"/>
- <use xlink:href="#DejaVuSans-46" transform="translate(820.046875 0)"/>
- <use xlink:href="#DejaVuSans-4b" transform="translate(875.03125 0)"/>
- <use xlink:href="#DejaVuSans-3" transform="translate(938.40625 0)"/>
- <use xlink:href="#DejaVuSans-33" transform="translate(970.1875 0)"/>
- <use xlink:href="#DejaVuSans-55" transform="translate(1028.734375 0)"/>
- <use xlink:href="#DejaVuSans-52" transform="translate(1067.640625 0)"/>
- <use xlink:href="#DejaVuSans-11" transform="translate(1127.078125 0)"/>
- <use xlink:href="#DejaVuSans-3" transform="translate(1158.859375 0)"/>
- <use xlink:href="#DejaVuSans-24" transform="translate(1190.640625 0)"/>
- <use xlink:href="#DejaVuSans-2c" transform="translate(1259.046875 0)"/>
- <use xlink:href="#DejaVuSans-30" transform="translate(1288.546875 0)"/>
- <use xlink:href="#DejaVuSans-28" transform="translate(1374.828125 0)"/>
- <use xlink:href="#DejaVuSans-15" transform="translate(1438.015625 0)"/>
- <use xlink:href="#DejaVuSans-18" transform="translate(1501.640625 0)"/>
- <use xlink:href="#DejaVuSans-1d" transform="translate(1565.265625 0)"/>
- <use xlink:href="#DejaVuSans-3" transform="translate(1598.953125 0)"/>
- <use xlink:href="#DejaVuSans-24" transform="translate(1630.734375 0)"/>
- <use xlink:href="#DejaVuSans-2c" transform="translate(1699.140625 0)"/>
- <use xlink:href="#DejaVuSans-30" transform="translate(1728.640625 0)"/>
- <use xlink:href="#DejaVuSans-28" transform="translate(1814.921875 0)"/>
- <use xlink:href="#DejaVuSans-3" transform="translate(1878.109375 0)"/>
- <use xlink:href="#DejaVuSans-15" transform="translate(1909.890625 0)"/>
- <use xlink:href="#DejaVuSans-13" transform="translate(1973.515625 0)"/>
- <use xlink:href="#DejaVuSans-15" transform="translate(2037.140625 0)"/>
- <use xlink:href="#DejaVuSans-18" transform="translate(2100.765625 0)"/>
- <use xlink:href="#DejaVuSans-11" transform="translate(2164.390625 0)"/>
- </g>
- </g>
<g id="line2d_5">
<path d="M 691.2 14.4
L 691.2 341.28
-" clip-path="url(#pdee8638843)" style="fill: none; stroke: #aaaaaa; stroke-width: 0.65; stroke-linecap: square"/>
+" clip-path="url(#p101463c74f)" style="fill: none; stroke: #aaaaaa; stroke-width: 0.65; stroke-linecap: square"/>
</g>
</g>
</g>
<defs>
- <clipPath id="pdee8638843">
- <rect x="0" y="0" width="864" height="406.8"/>
+ <clipPath id="p101463c74f">
+ <rect x="0" y="0" width="864" height="385.2"/>
</clipPath>
</defs>
</svg>