Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

x/create: quantize lm_head at 8-bit in the requested family

Jesse Gross83d4311ffecb
diff --git a/x/create/create.go b/x/create/create.go
index f1ef4feca..feca0c932 100644
--- a/x/create/create.go
+++ b/x/create/create.go
@@ -295,8 +295,12 @@ func GetTensorQuantization(name string, shape []int32, quantize string) string {
return ""
}
- // lm_head is too sensitive for the fp quant modes; keep it at source precision.
- if strings.HasSuffix(name, "lm_head.weight") && (quantNorm == "nvfp4" || quantNorm == "mxfp4" || quantNorm == "mxfp8") {
+ // lm_head is too sensitive for 4-bit types; the 8-bit type in the requested
+ // family keeps quality close to bf16 while saving decode bandwidth.
+ if strings.HasSuffix(name, "lm_head.weight") {
+ if e := eightBit(quantNorm); isAligned(shape, e) {
+ return e
+ }
return ""
}
diff --git a/x/create/create_test.go b/x/create/create_test.go
index ddd996487..4ba48b492 100644
--- a/x/create/create_test.go
+++ b/x/create/create_test.go
@@ -598,10 +598,10 @@ func TestGetTensorQuantization_MixedPrecisionPromotion(t *testing.T) {
// int8: already 8-bit, no promotion
{"v_proj int8 stays", "model.layers.0.self_attn.v_proj.weight", aligned, "int8", "int8"},
- // lm_head stays at source precision for fp modes, quantizes for affine
- {"lm_head nvfp4 kept", "lm_head.weight", aligned, "nvfp4", ""},
- {"lm_head mxfp8 kept", "lm_head.weight", aligned, "mxfp8", ""},
- {"lm_head int4 stays", "lm_head.weight", aligned, "int4", "int4"},
+ // lm_head resolves to the 8-bit type in the requested family
+ {"lm_head nvfp4 to mxfp8", "lm_head.weight", aligned, "nvfp4", "mxfp8"},
+ {"lm_head mxfp8 uniform", "lm_head.weight", aligned, "mxfp8", "mxfp8"},
+ {"lm_head int4 promoted", "lm_head.weight", aligned, "int4", "int8"},
// Expert tensors: down_proj also promoted for int4
{"expert down_proj int4", "model.layers.0.mlp.experts.down_proj.weight", []int32{128, 4096, 2816}, "int4", "int8"},