Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

llama: handle Gemma 4 and LFM2 BOS override in llama server (#16367)

Jeffrey Morganbe7de10c4144
diff --git a/llm/llama_server.go b/llm/llama_server.go
index 35ed31ce8..6d7358c7a 100644
--- a/llm/llama_server.go
+++ b/llm/llama_server.go
@@ -241,7 +241,21 @@ func (s *llamaServerRunner) tokenizerAddsBOS() bool {
return false
}
- return s.ggml.KV().Bool("tokenizer.ggml.add_bos_token")
+ kv := s.ggml.KV()
+
+ if kv.String("tokenizer.ggml.pre") == "lfm2" {
+ return true
+ }
+
+ // llama.cpp forces add_bos on for Gemma4 at load time, even for GGUFs
+ // whose tokenizer.ggml.add_bos_token metadata is explicitly false. Some
+ // GGUFs omit tokenizer.ggml.pre and are still treated as Gemma4 from
+ // tokenizer.ggml.model.
+ if kv.String("tokenizer.ggml.pre") == "gemma4" || kv.String("tokenizer.ggml.model") == "gemma4" {
+ return true
+ }
+
+ return kv.Bool("tokenizer.ggml.add_bos_token")
}
func (s *llamaServerRunner) completionPromptForRequest(ctx context.Context, req CompletionRequest) (any, error) {
diff --git a/llm/llama_server_test.go b/llm/llama_server_test.go
index c3230f141..2cb97ae3e 100644
--- a/llm/llama_server_test.go
+++ b/llm/llama_server_test.go
@@ -981,6 +981,7 @@ func TestLlamaServerCompletionBOSOwnership(t *testing.T) {
name string
leadingBOS string
tokenizerAddsBOS bool
+ ggmlKV ggml.KV
prompt string
wantPrompt string
}{
@@ -1020,6 +1021,59 @@ func TestLlamaServerCompletionBOSOwnership(t *testing.T) {
prompt: "<bos>hello<bos>",
wantPrompt: "hello<bos>",
},
+ {
+ name: "gemma4 llama.cpp runtime bos override",
+ leadingBOS: "<bos>",
+ ggmlKV: ggml.KV{
+ "general.architecture": "gemma4",
+ "tokenizer.ggml.pre": "gemma4",
+ "tokenizer.ggml.add_bos_token": false,
+ "tokenizer.ggml.bos_token_id": uint32(2),
+ "tokenizer.ggml.eos_token_id": uint32(1),
+ "tokenizer.ggml.unknown_token_id": uint32(0),
+ },
+ prompt: "<bos><|turn>user\nhello<turn|>\n<|turn>model\n",
+ wantPrompt: "<|turn>user\nhello<turn|>\n<|turn>model\n",
+ },
+ {
+ name: "gemma4 model runtime bos override",
+ leadingBOS: "<bos>",
+ ggmlKV: ggml.KV{
+ "general.architecture": "gemma4",
+ "tokenizer.ggml.model": "gemma4",
+ "tokenizer.ggml.add_bos_token": false,
+ "tokenizer.ggml.bos_token_id": uint32(2),
+ "tokenizer.ggml.eos_token_id": uint32(1),
+ "tokenizer.ggml.unknown_token_id": uint32(0),
+ },
+ prompt: "<bos><|turn>user\nhello<turn|>\n<|turn>model\n",
+ wantPrompt: "<|turn>user\nhello<turn|>\n<|turn>model\n",
+ },
+ {
+ name: "lfm2 strips renderer bos",
+ leadingBOS: "<|startoftext|>",
+ ggmlKV: ggml.KV{
+ "general.architecture": "lfm2",
+ "tokenizer.ggml.model": "gpt2",
+ "tokenizer.ggml.pre": "lfm2",
+ "tokenizer.ggml.add_bos_token": false,
+ "tokenizer.ggml.bos_token_id": uint32(0),
+ },
+ prompt: "<|startoftext|><|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n",
+ wantPrompt: "<|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n",
+ },
+ {
+ name: "lfm2 missing bos metadata uses llama.cpp default",
+ leadingBOS: "<|startoftext|>",
+ ggmlKV: ggml.KV{
+ "general.architecture": "lfm2",
+ "tokenizer.ggml.model": "gpt2",
+ "tokenizer.ggml.pre": "lfm2",
+ "tokenizer.ggml.bos_token_id": uint32(0),
+ },
+ prompt: "<|startoftext|><|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n",
+ wantPrompt: "<|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n",
+ },
}
for _, tt := range tests {
@@ -1053,7 +1107,9 @@ func TestLlamaServerCompletionBOSOwnership(t *testing.T) {
sem: semaphore.NewWeighted(1),
options: api.Options{Runner: api.Runner{NumCtx: 2048}},
}
- if tt.tokenizerAddsBOS {
+ if tt.ggmlKV != nil {
+ runner.ggml = loadTestGGML(t, tt.ggmlKV)
+ } else if tt.tokenizerAddsBOS {
runner.ggml = loadTestGGML(t, ggml.KV{
"general.architecture": "gemma3",
"tokenizer.ggml.add_bos_token": true,
diff --git a/model/renderers/cogito.go b/model/renderers/cogito.go
index f9c2b1d39..a6be9e23d 100644
--- a/model/renderers/cogito.go
+++ b/model/renderers/cogito.go
@@ -12,7 +12,7 @@ type CogitoRenderer struct {
}
func (r *CogitoRenderer) LeadingBOS() string {
- return ""
+ return "<|begin▁of▁sentence|>"
}
func (r *CogitoRenderer) Render(messages []api.Message, tools []api.Tool, thinkValue *api.ThinkValue) (string, error) {
diff --git a/model/renderers/renderer_test.go b/model/renderers/renderer_test.go
index b08658c41..5f9db37ee 100644
--- a/model/renderers/renderer_test.go
+++ b/model/renderers/renderer_test.go
@@ -70,6 +70,7 @@ func TestLeadingBOSForRenderer(t *testing.T) {
{name: "lfm2-thinking", want: "<|startoftext|>"},
{name: "laguna", want: "〈|EOS|〉"},
{name: "deepseek3.1", want: "<|begin▁of▁sentence|>"},
+ {name: "cogito", want: "<|begin▁of▁sentence|>"},
{name: "qwen3-coder", want: ""},
{name: "unknown", want: ""},
}