Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
x/create: quantize lm_head at 8-bit in the requested family
Jesse Gross83d4311ffecb
diff --git a/x/create/create.go b/x/create/create.goindex f1ef4feca..feca0c932 100644--- a/x/create/create.go+++ b/x/create/create.go@@ -295,8 +295,12 @@ func GetTensorQuantization(name string, shape []int32, quantize string) string {return ""}- // lm_head is too sensitive for the fp quant modes; keep it at source precision.- if strings.HasSuffix(name, "lm_head.weight") && (quantNorm == "nvfp4" || quantNorm == "mxfp4" || quantNorm == "mxfp8") {+ // lm_head is too sensitive for 4-bit types; the 8-bit type in the requested+ // family keeps quality close to bf16 while saving decode bandwidth.+ if strings.HasSuffix(name, "lm_head.weight") {+ if e := eightBit(quantNorm); isAligned(shape, e) {+ return e+ }return ""}diff --git a/x/create/create_test.go b/x/create/create_test.goindex ddd996487..4ba48b492 100644--- a/x/create/create_test.go+++ b/x/create/create_test.go@@ -598,10 +598,10 @@ func TestGetTensorQuantization_MixedPrecisionPromotion(t *testing.T) {// int8: already 8-bit, no promotion{"v_proj int8 stays", "model.layers.0.self_attn.v_proj.weight", aligned, "int8", "int8"},- // lm_head stays at source precision for fp modes, quantizes for affine- {"lm_head nvfp4 kept", "lm_head.weight", aligned, "nvfp4", ""},- {"lm_head mxfp8 kept", "lm_head.weight", aligned, "mxfp8", ""},- {"lm_head int4 stays", "lm_head.weight", aligned, "int4", "int4"},+ // lm_head resolves to the 8-bit type in the requested family+ {"lm_head nvfp4 to mxfp8", "lm_head.weight", aligned, "nvfp4", "mxfp8"},+ {"lm_head mxfp8 uniform", "lm_head.weight", aligned, "mxfp8", "mxfp8"},+ {"lm_head int4 promoted", "lm_head.weight", aligned, "int4", "int8"},// Expert tensors: down_proj also promoted for int4{"expert down_proj int4", "model.layers.0.mlp.experts.down_proj.weight", []int32{128, 4096, 2816}, "int4", "int8"},