Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

x/create: quantize a draft model's output head at the requested type

Jesse Gross132e0ca25d35
diff --git a/x/create/draft.go b/x/create/draft.go
index b6e3d9f51..759b2fd21 100644
--- a/x/create/draft.go
+++ b/x/create/draft.go
@@ -1,6 +1,9 @@
package create
-import "fmt"
+import (
+ "fmt"
+ "strings"
+)
// CreateDraftLayers imports a draft (speculative-decoding / MTP assistant)
// safetensors model into prefixed tensor and config blobs and returns the
@@ -69,14 +72,17 @@ func prefixSpecs(specs []BlobSpec, prefix string) []BlobSpec {
return out
}
-// draftPolicy wraps an architecture policy to keep a draft model's token
-// embedding at source precision (drafts start with unquantized embeddings; this
-// may change later). Every other tensor follows the wrapped policy. It is given
-// unprefixed source names, since planning runs before prefixSpecs.
+// draftPolicy wraps an architecture policy to give a draft model's output head
+// (tied token embedding or separate lm_head) the requested type directly: draft
+// quality only affects acceptance, so the target head's 8-bit promotion buys
+// nothing. It is given unprefixed source names; planning runs before prefixSpecs.
type draftPolicy struct{ inner quantizePolicy }
func (p draftPolicy) quantizationType(name string, shape []int32, requested string) string {
- if isEmbedTokensWeight(name) {
+ if isEmbedTokensWeight(name) || strings.HasSuffix(name, "lm_head.weight") {
+ if q := normalizeQuantType(requested); isAligned(shape, q) {
+ return q
+ }
return ""
}
return p.inner.quantizationType(name, shape, requested)
diff --git a/x/create/draft_test.go b/x/create/draft_test.go
index 6bf9ff67f..8e4ea4083 100644
--- a/x/create/draft_test.go
+++ b/x/create/draft_test.go
@@ -62,12 +62,19 @@ func TestPrefixSpecs(t *testing.T) {
}
}
-func TestDraftPolicyKeepsEmbeddingsUnquantized(t *testing.T) {
+func TestDraftPolicyQuantizesOutputHead(t *testing.T) {
p := draftPolicy{defaultQuantPolicy{}}
- // Draft token embeddings start unquantized regardless of the request.
- if got := p.quantizationType("model.embed_tokens.weight", []int32{4096, 2048}, "int8"); got != "" {
- t.Errorf("draft embed_tokens quant = %q, want \"\"", got)
+ // The output head takes the requested type: tied embedding or separate lm_head.
+ if got := p.quantizationType("model.embed_tokens.weight", []int32{4096, 2048}, "int8"); got != "int8" {
+ t.Errorf("draft embed_tokens quant = %q, want \"int8\"", got)
+ }
+ if got := p.quantizationType("lm_head.weight", []int32{4096, 2048}, "nvfp4"); got != "nvfp4" {
+ t.Errorf("draft lm_head quant = %q, want \"nvfp4\"", got)
+ }
+ // A head shape that does not fit the requested group stays at source precision.
+ if got := p.quantizationType("model.embed_tokens.weight", []int32{4096, 2049}, "int8"); got != "" {
+ t.Errorf("unaligned draft embed_tokens quant = %q, want \"\"", got)
}
// Other eligible weights still follow the wrapped policy.
if got := p.quantizationType("model.layers.0.mlp.down_proj.weight", []int32{2048, 2048}, "int8"); got == "" {