Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
llama-server: let llama-server infer the draft speculative type (#19064)
Daniel Hiltgene51196fe8c70
diff --git a/llm/llama_server.go b/llm/llama_server.goindex 0991bf377..5805edd64 100644--- a/llm/llama_server.go+++ b/llm/llama_server.go@@ -827,18 +827,20 @@ func appendContextShiftArgs(params []string, opts api.Options, enabled bool) []sconst (draftTypeMTP = "draft-mtp"- draftTypeDFlash = "draft-dflash"+ draftTypeEagle3 = "draft-eagle3")func appendDraftArgs(params []string, draftType, draftModelPath string, opts api.Options) []string {- if draftType == "" {+ if draftType == "" && draftModelPath == "" {return params}if opts.DraftNumPredict <= 0 {return params}- params = append(params, "--spec-type", draftType)+ if draftType != "" {+ params = append(params, "--spec-type", draftType)+ }params = append(params, "--spec-draft-n-max", strconv.Itoa(opts.DraftNumPredict))if draftType == draftTypeMTP {params = append(params, "--spec-draft-backend-sampling")@@ -849,15 +851,18 @@ func appendDraftArgs(params []string, draftType, draftModelPath string, opts apireturn params}-func externalDraftType(path string) (string, error) {- f, err := LoadModel(path, 1)+func externalDraftType(path string, shards ...string) (string, error) {+ f, err := LoadModel(path, 1, shards...)if err != nil {return "", fmt.Errorf("load draft model metadata: %w", err)}- if f.KV().Architecture() == "dflash" {- return draftTypeDFlash, nil+ if hasMTPDraft(f) {+ return draftTypeMTP, nil+ }+ if f.KV().Architecture() == "eagle3" {+ return draftTypeEagle3, nil}- return draftTypeMTP, nil+ return "", nil}func hasMTPDraft(f *gguf.Model) bool {@@ -934,7 +939,7 @@ func NewLlamaServerRunner(draftType = draftTypeMTP}if config.DraftModelPath != "" {- draftType, err = externalDraftType(config.DraftModelPath)+ draftType, err = externalDraftType(config.DraftModelPath, config.DraftModelShardPaths...)if err != nil {return nil, err}diff --git a/llm/llama_server_test.go b/llm/llama_server_test.goindex aed49b1a4..68c79856b 100644--- a/llm/llama_server_test.go+++ b/llm/llama_server_test.go@@ -2610,15 +2610,20 @@ func TestAppendDraftArgs(t *testing.T) {want: []string{"base", "--spec-type", "draft-mtp", "--spec-draft-n-max", "8", "--spec-draft-backend-sampling", "--spec-draft-model", "draft.gguf"},},{- name: "DFlash draft omits MTP backend sampling",- draftType: draftTypeDFlash,+ name: "Eagle3 draft passes its type without MTP backend sampling",+ draftType: draftTypeEagle3,draftPath: "draft.gguf",opts: api.Options{Runner: api.Runner{DraftNumPredict: 4}},- want: []string{"base", "--spec-type", "draft-dflash", "--spec-draft-n-max", "4", "--spec-draft-model", "draft.gguf"},+ want: []string{"base", "--spec-type", "draft-eagle3", "--spec-draft-n-max", "4", "--spec-draft-model", "draft.gguf"},+ },+ {+ name: "untyped draft leaves the type to llama-server",+ draftPath: "draft.gguf",+ opts: api.Options{Runner: api.Runner{DraftNumPredict: 4}},+ want: []string{"base", "--spec-draft-n-max", "4", "--spec-draft-model", "draft.gguf"},},{name: "zero draft depth disables speculative decoding",- draftType: draftTypeDFlash,draftPath: "draft.gguf",opts: api.Options{Runner: api.Runner{DraftNumPredict: 0}},want: []string{"base"},@@ -2637,17 +2642,46 @@ func TestAppendDraftArgs(t *testing.T) {func TestExternalDraftType(t *testing.T) {tests := []struct {- architecture string- want string+ name string+ kv gguftest.KV+ tensors []*gguftest.Tensor+ want string}{- {architecture: "dflash", want: draftTypeDFlash},- {architecture: "qwen35", want: draftTypeMTP},- {architecture: "unknown", want: draftTypeMTP},+ {+ name: "dflash family is inferred by llama-server",+ kv: gguftest.KV{"general.architecture": "dflash"},+ },+ {+ name: "eagle3",+ kv: gguftest.KV{"general.architecture": "eagle3"},+ want: draftTypeEagle3,+ },+ {+ name: "MTP sidecar by nextn key",+ kv: gguftest.KV{"general.architecture": "gemma4-assistant", "gemma4-assistant.nextn_predict_layers": uint32(4)},+ want: draftTypeMTP,+ },+ {+ name: "legacy qwen35 MTP sidecar",+ kv: gguftest.KV{"general.architecture": "qwen35"},+ tensors: []*gguftest.Tensor{+ testGGUFTensor("mtp.fc.weight", gguf.TensorTypeF32, []uint64{1}),+ },+ want: draftTypeMTP,+ },+ {+ name: "qwen35 without MTP markers is inferred by llama-server",+ kv: gguftest.KV{"general.architecture": "qwen35"},+ },+ {+ name: "unknown",+ kv: gguftest.KV{"general.architecture": "unknown"},+ },}for _, tt := range tests {- t.Run(tt.architecture, func(t *testing.T) {- path, _ := writeTestGGUF(t, gguftest.KV{"general.architecture": tt.architecture}, nil)+ t.Run(tt.name, func(t *testing.T) {+ path, _ := writeTestGGUF(t, tt.kv, tt.tensors)got, err := externalDraftType(path)if err != nil {t.Fatal(err)@@ -2657,6 +2691,20 @@ func TestExternalDraftType(t *testing.T) {}})}++ t.Run("MTP marker in a later shard", func(t *testing.T) {+ path, _ := writeTestGGUF(t, gguftest.KV{"general.architecture": "qwen35"}, nil)+ shard, _ := writeTestGGUF(t, gguftest.KV{"general.architecture": "unknown"}, []*gguftest.Tensor{+ testGGUFTensor("mtp.fc.weight", gguf.TensorTypeF32, []uint64{1}),+ })+ got, err := externalDraftType(path, shard)+ if err != nil {+ t.Fatal(err)+ }+ if got != draftTypeMTP {+ t.Fatalf("externalDraftType = %q, want %q", got, draftTypeMTP)+ }+ })}func TestHasLegacyQwenMTPDraft(t *testing.T) {