Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
x/create: quantize a draft model's output head at the requested type
Jesse Gross132e0ca25d35
diff --git a/x/create/draft.go b/x/create/draft.goindex b6e3d9f51..759b2fd21 100644--- a/x/create/draft.go+++ b/x/create/draft.go@@ -1,6 +1,9 @@package create-import "fmt"+import (+ "fmt"+ "strings"+)// CreateDraftLayers imports a draft (speculative-decoding / MTP assistant)// safetensors model into prefixed tensor and config blobs and returns the@@ -69,14 +72,17 @@ func prefixSpecs(specs []BlobSpec, prefix string) []BlobSpec {return out}-// draftPolicy wraps an architecture policy to keep a draft model's token-// embedding at source precision (drafts start with unquantized embeddings; this-// may change later). Every other tensor follows the wrapped policy. It is given-// unprefixed source names, since planning runs before prefixSpecs.+// draftPolicy wraps an architecture policy to give a draft model's output head+// (tied token embedding or separate lm_head) the requested type directly: draft+// quality only affects acceptance, so the target head's 8-bit promotion buys+// nothing. It is given unprefixed source names; planning runs before prefixSpecs.type draftPolicy struct{ inner quantizePolicy }func (p draftPolicy) quantizationType(name string, shape []int32, requested string) string {- if isEmbedTokensWeight(name) {+ if isEmbedTokensWeight(name) || strings.HasSuffix(name, "lm_head.weight") {+ if q := normalizeQuantType(requested); isAligned(shape, q) {+ return q+ }return ""}return p.inner.quantizationType(name, shape, requested)diff --git a/x/create/draft_test.go b/x/create/draft_test.goindex 6bf9ff67f..8e4ea4083 100644--- a/x/create/draft_test.go+++ b/x/create/draft_test.go@@ -62,12 +62,19 @@ func TestPrefixSpecs(t *testing.T) {}}-func TestDraftPolicyKeepsEmbeddingsUnquantized(t *testing.T) {+func TestDraftPolicyQuantizesOutputHead(t *testing.T) {p := draftPolicy{defaultQuantPolicy{}}- // Draft token embeddings start unquantized regardless of the request.- if got := p.quantizationType("model.embed_tokens.weight", []int32{4096, 2048}, "int8"); got != "" {- t.Errorf("draft embed_tokens quant = %q, want \"\"", got)+ // The output head takes the requested type: tied embedding or separate lm_head.+ if got := p.quantizationType("model.embed_tokens.weight", []int32{4096, 2048}, "int8"); got != "int8" {+ t.Errorf("draft embed_tokens quant = %q, want \"int8\"", got)+ }+ if got := p.quantizationType("lm_head.weight", []int32{4096, 2048}, "nvfp4"); got != "nvfp4" {+ t.Errorf("draft lm_head quant = %q, want \"nvfp4\"", got)+ }+ // A head shape that does not fit the requested group stays at source precision.+ if got := p.quantizationType("model.embed_tokens.weight", []int32{4096, 2049}, "int8"); got != "" {+ t.Errorf("unaligned draft embed_tokens quant = %q, want \"\"", got)}// Other eligible weights still follow the wrapped policy.if got := p.quantizationType("model.layers.0.mlp.down_proj.weight", []int32{2048, 2048}, "int8"); got == "" {