Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

llama-server: let llama-server infer the draft speculative type (#19064)

Daniel Hiltgene51196fe8c70
diff --git a/llm/llama_server.go b/llm/llama_server.go
index 0991bf377..5805edd64 100644
--- a/llm/llama_server.go
+++ b/llm/llama_server.go
@@ -827,18 +827,20 @@ func appendContextShiftArgs(params []string, opts api.Options, enabled bool) []s
const (
draftTypeMTP = "draft-mtp"
- draftTypeDFlash = "draft-dflash"
+ draftTypeEagle3 = "draft-eagle3"
)
func appendDraftArgs(params []string, draftType, draftModelPath string, opts api.Options) []string {
- if draftType == "" {
+ if draftType == "" && draftModelPath == "" {
return params
}
if opts.DraftNumPredict <= 0 {
return params
}
- params = append(params, "--spec-type", draftType)
+ if draftType != "" {
+ params = append(params, "--spec-type", draftType)
+ }
params = append(params, "--spec-draft-n-max", strconv.Itoa(opts.DraftNumPredict))
if draftType == draftTypeMTP {
params = append(params, "--spec-draft-backend-sampling")
@@ -849,15 +851,18 @@ func appendDraftArgs(params []string, draftType, draftModelPath string, opts api
return params
}
-func externalDraftType(path string) (string, error) {
- f, err := LoadModel(path, 1)
+func externalDraftType(path string, shards ...string) (string, error) {
+ f, err := LoadModel(path, 1, shards...)
if err != nil {
return "", fmt.Errorf("load draft model metadata: %w", err)
}
- if f.KV().Architecture() == "dflash" {
- return draftTypeDFlash, nil
+ if hasMTPDraft(f) {
+ return draftTypeMTP, nil
+ }
+ if f.KV().Architecture() == "eagle3" {
+ return draftTypeEagle3, nil
}
- return draftTypeMTP, nil
+ return "", nil
}
func hasMTPDraft(f *gguf.Model) bool {
@@ -934,7 +939,7 @@ func NewLlamaServerRunner(
draftType = draftTypeMTP
}
if config.DraftModelPath != "" {
- draftType, err = externalDraftType(config.DraftModelPath)
+ draftType, err = externalDraftType(config.DraftModelPath, config.DraftModelShardPaths...)
if err != nil {
return nil, err
}
diff --git a/llm/llama_server_test.go b/llm/llama_server_test.go
index aed49b1a4..68c79856b 100644
--- a/llm/llama_server_test.go
+++ b/llm/llama_server_test.go
@@ -2610,15 +2610,20 @@ func TestAppendDraftArgs(t *testing.T) {
want: []string{"base", "--spec-type", "draft-mtp", "--spec-draft-n-max", "8", "--spec-draft-backend-sampling", "--spec-draft-model", "draft.gguf"},
},
{
- name: "DFlash draft omits MTP backend sampling",
- draftType: draftTypeDFlash,
+ name: "Eagle3 draft passes its type without MTP backend sampling",
+ draftType: draftTypeEagle3,
draftPath: "draft.gguf",
opts: api.Options{Runner: api.Runner{DraftNumPredict: 4}},
- want: []string{"base", "--spec-type", "draft-dflash", "--spec-draft-n-max", "4", "--spec-draft-model", "draft.gguf"},
+ want: []string{"base", "--spec-type", "draft-eagle3", "--spec-draft-n-max", "4", "--spec-draft-model", "draft.gguf"},
+ },
+ {
+ name: "untyped draft leaves the type to llama-server",
+ draftPath: "draft.gguf",
+ opts: api.Options{Runner: api.Runner{DraftNumPredict: 4}},
+ want: []string{"base", "--spec-draft-n-max", "4", "--spec-draft-model", "draft.gguf"},
},
{
name: "zero draft depth disables speculative decoding",
- draftType: draftTypeDFlash,
draftPath: "draft.gguf",
opts: api.Options{Runner: api.Runner{DraftNumPredict: 0}},
want: []string{"base"},
@@ -2637,17 +2642,46 @@ func TestAppendDraftArgs(t *testing.T) {
func TestExternalDraftType(t *testing.T) {
tests := []struct {
- architecture string
- want string
+ name string
+ kv gguftest.KV
+ tensors []*gguftest.Tensor
+ want string
}{
- {architecture: "dflash", want: draftTypeDFlash},
- {architecture: "qwen35", want: draftTypeMTP},
- {architecture: "unknown", want: draftTypeMTP},
+ {
+ name: "dflash family is inferred by llama-server",
+ kv: gguftest.KV{"general.architecture": "dflash"},
+ },
+ {
+ name: "eagle3",
+ kv: gguftest.KV{"general.architecture": "eagle3"},
+ want: draftTypeEagle3,
+ },
+ {
+ name: "MTP sidecar by nextn key",
+ kv: gguftest.KV{"general.architecture": "gemma4-assistant", "gemma4-assistant.nextn_predict_layers": uint32(4)},
+ want: draftTypeMTP,
+ },
+ {
+ name: "legacy qwen35 MTP sidecar",
+ kv: gguftest.KV{"general.architecture": "qwen35"},
+ tensors: []*gguftest.Tensor{
+ testGGUFTensor("mtp.fc.weight", gguf.TensorTypeF32, []uint64{1}),
+ },
+ want: draftTypeMTP,
+ },
+ {
+ name: "qwen35 without MTP markers is inferred by llama-server",
+ kv: gguftest.KV{"general.architecture": "qwen35"},
+ },
+ {
+ name: "unknown",
+ kv: gguftest.KV{"general.architecture": "unknown"},
+ },
}
for _, tt := range tests {
- t.Run(tt.architecture, func(t *testing.T) {
- path, _ := writeTestGGUF(t, gguftest.KV{"general.architecture": tt.architecture}, nil)
+ t.Run(tt.name, func(t *testing.T) {
+ path, _ := writeTestGGUF(t, tt.kv, tt.tensors)
got, err := externalDraftType(path)
if err != nil {
t.Fatal(err)
@@ -2657,6 +2691,20 @@ func TestExternalDraftType(t *testing.T) {
}
})
}
+
+ t.Run("MTP marker in a later shard", func(t *testing.T) {
+ path, _ := writeTestGGUF(t, gguftest.KV{"general.architecture": "qwen35"}, nil)
+ shard, _ := writeTestGGUF(t, gguftest.KV{"general.architecture": "unknown"}, []*gguftest.Tensor{
+ testGGUFTensor("mtp.fc.weight", gguf.TensorTypeF32, []uint64{1}),
+ })
+ got, err := externalDraftType(path, shard)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != draftTypeMTP {
+ t.Fatalf("externalDraftType = %q, want %q", got, draftTypeMTP)
+ }
+ })
}
func TestHasLegacyQwenMTPDraft(t *testing.T) {