Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
llama: handle Gemma 4 and LFM2 BOS override in llama server (#16367)
Jeffrey Morganbe7de10c4144
diff --git a/llm/llama_server.go b/llm/llama_server.goindex 35ed31ce8..6d7358c7a 100644--- a/llm/llama_server.go+++ b/llm/llama_server.go@@ -241,7 +241,21 @@ func (s *llamaServerRunner) tokenizerAddsBOS() bool {return false}- return s.ggml.KV().Bool("tokenizer.ggml.add_bos_token")+ kv := s.ggml.KV()++ if kv.String("tokenizer.ggml.pre") == "lfm2" {+ return true+ }++ // llama.cpp forces add_bos on for Gemma4 at load time, even for GGUFs+ // whose tokenizer.ggml.add_bos_token metadata is explicitly false. Some+ // GGUFs omit tokenizer.ggml.pre and are still treated as Gemma4 from+ // tokenizer.ggml.model.+ if kv.String("tokenizer.ggml.pre") == "gemma4" || kv.String("tokenizer.ggml.model") == "gemma4" {+ return true+ }++ return kv.Bool("tokenizer.ggml.add_bos_token")}func (s *llamaServerRunner) completionPromptForRequest(ctx context.Context, req CompletionRequest) (any, error) {diff --git a/llm/llama_server_test.go b/llm/llama_server_test.goindex c3230f141..2cb97ae3e 100644--- a/llm/llama_server_test.go+++ b/llm/llama_server_test.go@@ -981,6 +981,7 @@ func TestLlamaServerCompletionBOSOwnership(t *testing.T) {name stringleadingBOS stringtokenizerAddsBOS bool+ ggmlKV ggml.KVprompt stringwantPrompt string}{@@ -1020,6 +1021,59 @@ func TestLlamaServerCompletionBOSOwnership(t *testing.T) {prompt: "<bos>hello<bos>",wantPrompt: "hello<bos>",},+ {+ name: "gemma4 llama.cpp runtime bos override",+ leadingBOS: "<bos>",+ ggmlKV: ggml.KV{+ "general.architecture": "gemma4",+ "tokenizer.ggml.pre": "gemma4",+ "tokenizer.ggml.add_bos_token": false,+ "tokenizer.ggml.bos_token_id": uint32(2),+ "tokenizer.ggml.eos_token_id": uint32(1),+ "tokenizer.ggml.unknown_token_id": uint32(0),+ },+ prompt: "<bos><|turn>user\nhello<turn|>\n<|turn>model\n",+ wantPrompt: "<|turn>user\nhello<turn|>\n<|turn>model\n",+ },+ {+ name: "gemma4 model runtime bos override",+ leadingBOS: "<bos>",+ ggmlKV: ggml.KV{+ "general.architecture": "gemma4",+ "tokenizer.ggml.model": "gemma4",+ "tokenizer.ggml.add_bos_token": false,+ "tokenizer.ggml.bos_token_id": uint32(2),+ "tokenizer.ggml.eos_token_id": uint32(1),+ "tokenizer.ggml.unknown_token_id": uint32(0),+ },+ prompt: "<bos><|turn>user\nhello<turn|>\n<|turn>model\n",+ wantPrompt: "<|turn>user\nhello<turn|>\n<|turn>model\n",+ },+ {+ name: "lfm2 strips renderer bos",+ leadingBOS: "<|startoftext|>",+ ggmlKV: ggml.KV{+ "general.architecture": "lfm2",+ "tokenizer.ggml.model": "gpt2",+ "tokenizer.ggml.pre": "lfm2",+ "tokenizer.ggml.add_bos_token": false,+ "tokenizer.ggml.bos_token_id": uint32(0),+ },+ prompt: "<|startoftext|><|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n",+ wantPrompt: "<|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n",+ },+ {+ name: "lfm2 missing bos metadata uses llama.cpp default",+ leadingBOS: "<|startoftext|>",+ ggmlKV: ggml.KV{+ "general.architecture": "lfm2",+ "tokenizer.ggml.model": "gpt2",+ "tokenizer.ggml.pre": "lfm2",+ "tokenizer.ggml.bos_token_id": uint32(0),+ },+ prompt: "<|startoftext|><|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n",+ wantPrompt: "<|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n",+ },}for _, tt := range tests {@@ -1053,7 +1107,9 @@ func TestLlamaServerCompletionBOSOwnership(t *testing.T) {sem: semaphore.NewWeighted(1),options: api.Options{Runner: api.Runner{NumCtx: 2048}},}- if tt.tokenizerAddsBOS {+ if tt.ggmlKV != nil {+ runner.ggml = loadTestGGML(t, tt.ggmlKV)+ } else if tt.tokenizerAddsBOS {runner.ggml = loadTestGGML(t, ggml.KV{"general.architecture": "gemma3","tokenizer.ggml.add_bos_token": true,diff --git a/model/renderers/cogito.go b/model/renderers/cogito.goindex f9c2b1d39..a6be9e23d 100644--- a/model/renderers/cogito.go+++ b/model/renderers/cogito.go@@ -12,7 +12,7 @@ type CogitoRenderer struct {}func (r *CogitoRenderer) LeadingBOS() string {- return ""+ return "<|begin▁of▁sentence|>"}func (r *CogitoRenderer) Render(messages []api.Message, tools []api.Tool, thinkValue *api.ThinkValue) (string, error) {diff --git a/model/renderers/renderer_test.go b/model/renderers/renderer_test.goindex b08658c41..5f9db37ee 100644--- a/model/renderers/renderer_test.go+++ b/model/renderers/renderer_test.go@@ -70,6 +70,7 @@ func TestLeadingBOSForRenderer(t *testing.T) {{name: "lfm2-thinking", want: "<|startoftext|>"},{name: "laguna", want: "〈|EOS|〉"},{name: "deepseek3.1", want: "<|begin▁of▁sentence|>"},+ {name: "cogito", want: "<|begin▁of▁sentence|>"},{name: "qwen3-coder", want: ""},{name: "unknown", want: ""},}