Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

llm: apply a format after the thinking in one llama-server request

Jesse Gross2ff052b78108
diff --git a/llm/gbnf.go b/llm/gbnf.go
new file mode 100644
index 000000000..8703386b2
--- /dev/null
+++ b/llm/gbnf.go
@@ -0,0 +1,180 @@
+package llm
+
+import (
+ "fmt"
+ "slices"
+ "strings"
+ "unicode"
+)
+
+// thinkingGrammar returns a grammar that leaves the text before any of the
+// closings unconstrained, then constrains what follows the first complete
+// closing to the root of the format grammar. The response may end before any
+// closing. A closing that is a prefix of another ends the thinking as soon
+// as it is complete.
+func thinkingGrammar(closings []string, format string) string {
+ // The rules added here take a prefix no rule of the format grammar starts
+ // with.
+ prefix := "ollama-"
+ for grammarHasRulePrefix(format, prefix) {
+ prefix += "ollama-"
+ }
+ thinking, formatted := prefix+"thinking-", prefix+"format"
+
+ a := newClosingAutomaton(closings)
+ var b strings.Builder
+ fmt.Fprintf(&b, "root ::= %s0\n", thinking)
+ for state := range a.states {
+ fmt.Fprintf(&b, "%s%d ::=", thinking, state)
+ var matching, terminal []rune
+ next := map[int][]rune{}
+ for _, r := range a.alphabet {
+ to, done := a.next(state, r)
+ if done {
+ terminal = append(terminal, r)
+ } else if to != 0 {
+ next[to] = append(next[to], r)
+ }
+ if done || to != 0 {
+ matching = append(matching, r)
+ }
+ }
+ // Any other character restarts the search from the empty prefix.
+ fmt.Fprintf(&b, " | %s %s0", gbnfCharClass(matching, true), thinking)
+ for to := range a.states {
+ if rs := next[to]; len(rs) > 0 {
+ fmt.Fprintf(&b, " | %s %s%d", gbnfCharClass(rs, false), thinking, to)
+ }
+ }
+ if len(terminal) > 0 {
+ fmt.Fprintf(&b, " | %s %s", gbnfCharClass(terminal, false), formatted)
+ }
+ b.WriteByte('\n')
+ }
+ b.WriteString(renameGrammarRule(format, "root", formatted))
+ return b.String()
+}
+
+// closingAutomaton is the Aho-Corasick automaton over the closings: each
+// state is a proper prefix of a closing, state 0 the empty one.
+type closingAutomaton struct {
+ closings []string
+ states []string
+ index map[string]int
+ alphabet []rune
+}
+
+func newClosingAutomaton(closings []string) *closingAutomaton {
+ a := &closingAutomaton{closings: closings, index: map[string]int{}}
+ add := func(prefix string) {
+ if _, ok := a.index[prefix]; !ok {
+ a.index[prefix] = len(a.states)
+ a.states = append(a.states, prefix)
+ }
+ }
+ add("")
+ for _, closing := range closings {
+ for i := range closing {
+ add(closing[:i])
+ }
+ for _, r := range closing {
+ if !slices.Contains(a.alphabet, r) {
+ a.alphabet = append(a.alphabet, r)
+ }
+ }
+ }
+ slices.Sort(a.alphabet)
+ return a
+}
+
+// next returns the state after reading r in state, or done when the text
+// read so far ends with a closing.
+func (a *closingAutomaton) next(state int, r rune) (to int, done bool) {
+ s := a.states[state] + string(r)
+ for _, closing := range a.closings {
+ if strings.HasSuffix(s, closing) {
+ return 0, true
+ }
+ }
+ for i := range s {
+ if to, ok := a.index[s[i:]]; ok {
+ return to, false
+ }
+ }
+ return 0, false
+}
+
+func gbnfCharClass(rs []rune, negate bool) string {
+ var b strings.Builder
+ b.WriteByte('[')
+ if negate {
+ b.WriteByte('^')
+ }
+ for _, r := range rs {
+ switch {
+ case r > unicode.MaxASCII || !unicode.IsPrint(r) || strings.ContainsRune(`]\^-`, r):
+ fmt.Fprintf(&b, `\U%08X`, r)
+ default:
+ b.WriteRune(r)
+ }
+ }
+ b.WriteByte(']')
+ return b.String()
+}
+
+func renameGrammarRule(grammar, from, to string) string {
+ var b strings.Builder
+ scanGrammar(grammar, func(text string, name bool) {
+ if name && text == from {
+ text = to
+ }
+ b.WriteString(text)
+ })
+ return b.String()
+}
+
+func grammarHasRulePrefix(grammar, prefix string) bool {
+ found := false
+ scanGrammar(grammar, func(text string, name bool) {
+ found = found || name && strings.HasPrefix(text, prefix)
+ })
+ return found
+}
+
+// scanGrammar splits a grammar into rule names and everything else. Comments,
+// quoted strings and character classes are passed whole, so a word inside
+// them is never taken for a name.
+func scanGrammar(grammar string, fn func(text string, name bool)) {
+ isName := func(c byte) bool {
+ return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-' || c == '_'
+ }
+ for i := 0; i < len(grammar); {
+ start := i
+ switch c := grammar[i]; {
+ case c == '#':
+ for i < len(grammar) && grammar[i] != '\n' {
+ i++
+ }
+ case c == '"' || c == '[':
+ end := byte('"')
+ if c == '[' {
+ end = ']'
+ }
+ for i++; i < len(grammar) && grammar[i] != end; i++ {
+ if grammar[i] == '\\' {
+ i++
+ }
+ }
+ i = min(i+1, len(grammar))
+ case isName(c):
+ for i < len(grammar) && isName(grammar[i]) {
+ i++
+ }
+ fn(grammar[start:i], true)
+ continue
+ default:
+ i++
+ }
+ fn(grammar[start:i], false)
+ }
+}
diff --git a/llm/gbnf_test.go b/llm/gbnf_test.go
new file mode 100644
index 000000000..4ae1456f9
--- /dev/null
+++ b/llm/gbnf_test.go
@@ -0,0 +1,79 @@
+package llm
+
+import (
+ "strings"
+ "testing"
+)
+
+// TestRenameGrammarRule checks that only rule names change, not the same word
+// in a comment, a string, a character class or a longer name.
+func TestRenameGrammarRule(t *testing.T) {
+ grammar := "# root stays in a comment\n" +
+ "root ::= \"root\" | [root] | \"[\" root \"]\" | root-item\n" +
+ "root-item ::= \"\\\"root\\\"\" root\n"
+ want := "# root stays in a comment\n" +
+ "out ::= \"root\" | [root] | \"[\" out \"]\" | root-item\n" +
+ "root-item ::= \"\\\"root\\\"\" out\n"
+ if got := renameGrammarRule(grammar, "root", "out"); got != want {
+ t.Errorf("renamed grammar:\n%s\nwant:\n%s", got, want)
+ }
+}
+
+// TestClosingAutomatonNext checks that the state after a character is the
+// longest suffix of the text so far that is a proper prefix of a closing, and
+// that the automaton is done exactly when the text ends with a closing. The
+// closings share a prefix and overlap, so a reset to the empty prefix on a
+// mismatch would lose matches.
+func TestClosingAutomatonNext(t *testing.T) {
+ a := newClosingAutomaton([]string{"<|x|>", "<|y|>", "y|>|"})
+ state := 0
+ for _, step := range []struct {
+ r rune
+ want string
+ done bool
+ }{
+ {r: '<', want: "<"},
+ {r: '|', want: "<|"},
+ {r: '<', want: "<"},
+ {r: '|', want: "<|"},
+ {r: 'y', want: "<|y"},
+ {r: '|', want: "<|y|"},
+ {r: '>', done: true},
+ {r: '|', want: ""},
+ {r: 'y', want: "y"},
+ {r: '|', want: "y|"},
+ {r: '>', want: "y|>"},
+ {r: '<', want: "<"},
+ {r: 'z', want: ""},
+ } {
+ to, done := a.next(state, step.r)
+ if done != step.done || (!done && a.states[to] != step.want) {
+ t.Fatalf("after %q: state %q done %v, want state %q done %v", step.r, a.states[to], done, step.want, step.done)
+ }
+ state = to
+ }
+}
+
+// TestThinkingGrammarRules checks the emitted rules: every prefix state may
+// end the response, completing the closing leads to the renamed format root,
+// the format grammar's other rules keep their names, and the added rules
+// avoid a prefix the format grammar already uses.
+func TestThinkingGrammarRules(t *testing.T) {
+ format := "root ::= \"{\" space \"}\"\nspace ::= | \" \"\n"
+ grammar := thinkingGrammar([]string{"</think>"}, format)
+ for _, want := range []string{
+ "root ::= ollama-thinking-0\n",
+ "ollama-thinking-0 ::= | [^<] ollama-thinking-0 | [<] ollama-thinking-1\n",
+ "ollama-thinking-7 ::= | [^<>] ollama-thinking-0 | [<] ollama-thinking-1 | [>] ollama-format\n",
+ "ollama-format ::= \"{\" space \"}\"\nspace ::= | \" \"\n",
+ } {
+ if !strings.Contains(grammar, want) {
+ t.Errorf("grammar lacks %q:\n%s", want, grammar)
+ }
+ }
+
+ grammar = thinkingGrammar([]string{"</think>"}, "root ::= ollama-x\nollama-x ::= \"{}\"\n")
+ if !strings.HasPrefix(grammar, "root ::= ollama-ollama-thinking-0\n") || !strings.Contains(grammar, "ollama-ollama-format ::= ollama-x\n") {
+ t.Errorf("grammar does not avoid the format's rule prefix:\n%s", grammar)
+ }
+}
diff --git a/llm/llama_server.go b/llm/llama_server.go
index 7a0b1ed33..c4eb4adb0 100644
--- a/llm/llama_server.go
+++ b/llm/llama_server.go
@@ -5,9 +5,10 @@
// still render prompts in Go and call /completion. Other GGUF chat models use
// llama-server's chat_template handling through /v1/chat/completions.
//
-// For structured output, JSON schemas are passed directly to llama-server via
-// its json_schema field (avoiding the CGO SchemaToGrammar dependency). Raw BNF
-// grammars are passed via the grammar field.
+// For structured output, a JSON schema is passed to llama-server via its
+// json_schema field and the "json" format as a builtin grammar via the grammar
+// field. A format that applies after a response's thinking is sent as a
+// grammar built around the GBNF llama-server itself derives from the schema.
//
// llama-server auto-detects GPU layers (-ngl), thread count (-t), and flash
// attention (--flash-attn).
@@ -16,6 +17,7 @@ package llm
import (
"bufio"
"bytes"
+ "container/list"
"context"
crand "crypto/rand"
"encoding/base64"
@@ -1425,6 +1427,85 @@ type llamaServerCompletionRequest struct {
TimingsPerToken bool `json:"timings_per_token,omitempty"`
}
+// schemaGrammars caches the grammars llama-server derives from JSON schemas,
+// most recently used first. The conversion depends only on the llama-server
+// build, which is fixed for the process, so entries are shared by every
+// runner and never expire.
+var schemaGrammars = struct {
+ sync.Mutex
+ entries map[string]*list.Element
+ order list.List
+}{entries: map[string]*list.Element{}}
+
+const schemaGrammarsSize = 64
+
+type schemaGrammar struct {
+ schema, grammar string
+}
+
+// schemaGrammar converts a JSON schema to GBNF with llama-server's own
+// converter. An empty completion evaluates and generates nothing, but its
+// final response still reports the grammar the schema was converted to.
+func (s *llamaServerRunner) schemaGrammar(ctx context.Context, schema json.RawMessage) (string, error) {
+ key := string(schema)
+ schemaGrammars.Lock()
+ if e, ok := schemaGrammars.entries[key]; ok {
+ schemaGrammars.order.MoveToFront(e)
+ schemaGrammars.Unlock()
+ return e.Value.(*schemaGrammar).grammar, nil
+ }
+ schemaGrammars.Unlock()
+
+ body, err := json.Marshal(struct {
+ Prompt [][]int `json:"prompt"`
+ NPredict int `json:"n_predict"`
+ JsonSchema json.RawMessage `json:"json_schema"`
+ ResponseFields []string `json:"response_fields"`
+ }{Prompt: [][]int{{}}, JsonSchema: schema, ResponseFields: []string{"generation_settings/grammar"}})
+ if err != nil {
+ return "", fmt.Errorf("failed to marshal grammar request: %v", err)
+ }
+ endpoint := fmt.Sprintf("http://127.0.0.1:%d/completion", s.port)
+ serverReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
+ if err != nil {
+ return "", fmt.Errorf("error creating grammar request: %v", err)
+ }
+ serverReq.Header.Set("Content-Type", "application/json")
+ res, err := s.httpClient().Do(serverReq)
+ if err != nil {
+ return "", fmt.Errorf("llama-server grammar request failed: %v", err)
+ }
+ defer res.Body.Close()
+ resBody, err := io.ReadAll(res.Body)
+ if err != nil {
+ return "", fmt.Errorf("failed reading llama-server grammar response: %w", err)
+ }
+ if res.StatusCode >= 400 {
+ return "", api.StatusError{StatusCode: res.StatusCode, ErrorMessage: s.statusErrorMessage(resBody)}
+ }
+ var lsRes struct {
+ Grammar string `json:"generation_settings/grammar"`
+ }
+ if err := json.Unmarshal(resBody, &lsRes); err != nil {
+ return "", fmt.Errorf("error unmarshalling llama-server grammar response: %v", err)
+ }
+ if lsRes.Grammar == "" {
+ return "", errors.New("llama-server returned no grammar for the schema")
+ }
+
+ schemaGrammars.Lock()
+ defer schemaGrammars.Unlock()
+ if _, ok := schemaGrammars.entries[key]; !ok {
+ if schemaGrammars.order.Len() == schemaGrammarsSize {
+ oldest := schemaGrammars.order.Back()
+ delete(schemaGrammars.entries, oldest.Value.(*schemaGrammar).schema)
+ schemaGrammars.order.Remove(oldest)
+ }
+ schemaGrammars.entries[key] = schemaGrammars.order.PushFront(&schemaGrammar{schema: key, grammar: lsRes.Grammar})
+ }
+ return lsRes.Grammar, nil
+}
+
func llamaServerPreservedTokens(parserTokens []string, toolCallTag string) []string {
tokens := append([]string{}, parserTokens...)
tokens = append(tokens, llamaServerPreservedTokensForToolTag(toolCallTag)...)
@@ -1611,6 +1692,19 @@ func (s *llamaServerRunner) Completion(ctx context.Context, req CompletionReques
}
}
+ // A format on a thinking response applies after the closing string, which
+ // only a grammar of our own can express, so a schema is converted first.
+ if len(req.ThinkingClose) > 0 && (lsReq.Grammar != "" || lsReq.JsonSchema != nil) {
+ if lsReq.Grammar == "" {
+ grammar, err := s.schemaGrammar(ctx, lsReq.JsonSchema)
+ if err != nil {
+ return err
+ }
+ lsReq.Grammar, lsReq.JsonSchema = grammar, nil
+ }
+ lsReq.Grammar = thinkingGrammar(req.ThinkingClose, lsReq.Grammar)
+ }
+
// Convert media: replace Ollama's stable [img-N] markers with the per-process
// llama-server media marker and package the matching payloads as base64.
if len(req.Media) > 0 {
diff --git a/llm/llama_server_test.go b/llm/llama_server_test.go
index 60193ff9c..7c761b348 100644
--- a/llm/llama_server_test.go
+++ b/llm/llama_server_test.go
@@ -3847,3 +3847,116 @@ func fakeRunningCmd() *exec.Cmd {
// SIGKILL children when the test process exits.
return cmd
}
+
+// TestLlamaServerCompletionThinkingFormat checks that a format on a thinking
+// response is one request carrying a grammar: a schema is converted by an
+// empty completion once per schema, "json" needs no conversion, the grammar
+// wraps the format rules behind the closing string, and content and metrics
+// pass through unchanged.
+func TestLlamaServerCompletionThinkingFormat(t *testing.T) {
+ schema := json.RawMessage(`{"type":"object"}`)
+ converted := "root ::= \"{\" space \"}\"\nspace ::= | \" \"\n"
+ sseLines := []string{
+ `data: {"content":"Let me think.</think>","stop":false,"timings":{"cache_n":2,"prompt_n":3,"prompt_ms":10,"predicted_n":3,"predicted_ms":15}}`,
+ `data: {"content":"{}","stop":false,"timings":{"cache_n":2,"prompt_n":3,"prompt_ms":10,"predicted_n":4,"predicted_ms":20}}`,
+ `data: {"content":"","stop":true,"stop_type":"eos","timings":{"cache_n":2,"prompt_n":3,"prompt_ms":10,"predicted_n":4,"predicted_ms":20}}`,
+ }
+
+ var conversions atomic.Int32
+ grammars := make(chan string, 8)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/health" {
+ fmt.Fprint(w, `{"status":"ok"}`)
+ return
+ }
+ var reqBody struct {
+ Prompt any `json:"prompt"`
+ NPredict *int `json:"n_predict"`
+ JsonSchema json.RawMessage `json:"json_schema"`
+ Grammar string `json:"grammar"`
+ ResponseFields []string `json:"response_fields"`
+ }
+ if err := json.NewDecoder(r.Body).Decode(&reqBody); err != nil {
+ t.Errorf("invalid request body: %v", err)
+ return
+ }
+ if len(reqBody.ResponseFields) > 0 {
+ conversions.Add(1)
+ if !reflect.DeepEqual(reqBody.Prompt, []any{[]any{}}) || reqBody.NPredict == nil || *reqBody.NPredict != 0 {
+ t.Errorf("conversion request prompt %v n_predict %v, want an empty token prompt and no generation", reqBody.Prompt, reqBody.NPredict)
+ }
+ if !bytes.Equal(reqBody.JsonSchema, schema) || !reflect.DeepEqual(reqBody.ResponseFields, []string{"generation_settings/grammar"}) {
+ t.Errorf("conversion request schema %s fields %v", reqBody.JsonSchema, reqBody.ResponseFields)
+ }
+ json.NewEncoder(w).Encode(map[string]string{"generation_settings/grammar": converted})
+ return
+ }
+ if reqBody.JsonSchema != nil {
+ t.Errorf("completion carried schema %s", reqBody.JsonSchema)
+ }
+ grammars <- reqBody.Grammar
+ w.Header().Set("Content-Type", "text/event-stream")
+ for _, line := range sseLines {
+ fmt.Fprintln(w, line)
+ fmt.Fprintln(w)
+ }
+ }))
+ defer srv.Close()
+
+ parts := strings.Split(srv.URL, ":")
+ var portInt int
+ fmt.Sscanf(parts[len(parts)-1], "%d", &portInt)
+
+ runner := &llamaServerRunner{
+ port: portInt,
+ cmd: fakeRunningCmd(),
+ sem: semaphore.NewWeighted(1),
+ options: api.Options{Runner: api.Runner{NumCtx: 2048}},
+ }
+
+ complete := func(format json.RawMessage) string {
+ var responses []CompletionResponse
+ opts := api.DefaultOptions()
+ err := runner.Completion(t.Context(), CompletionRequest{
+ Prompt: "test prompt",
+ Format: format,
+ ThinkingClose: []string{"</think>"},
+ Options: &opts,
+ }, func(cr CompletionResponse) {
+ responses = append(responses, cr)
+ })
+ if err != nil {
+ t.Fatalf("Completion error: %v", err)
+ }
+ var content strings.Builder
+ for _, resp := range responses[:len(responses)-1] {
+ content.WriteString(resp.Content)
+ }
+ if got := content.String(); got != "Let me think.</think>{}" {
+ t.Errorf("streamed content = %q", got)
+ }
+ final := responses[len(responses)-1]
+ if !final.Done || final.DoneReason != DoneReasonStop || final.EvalCount != 4 || final.PromptEvalCount != 5 {
+ t.Errorf("final response = %+v, want done with 4 generated and 5 prompt tokens", final)
+ }
+ return <-grammars
+ }
+
+ for range 2 {
+ grammar := complete(schema)
+ if !strings.HasPrefix(grammar, "root ::= ollama-thinking-0\n") || !strings.Contains(grammar, "ollama-format ::= \"{\" space \"}\"\n") {
+ t.Errorf("grammar does not wrap the converted schema:\n%s", grammar)
+ }
+ }
+ if got := conversions.Load(); got != 1 {
+ t.Errorf("schema converted %d times, want once", got)
+ }
+
+ grammar := complete(json.RawMessage(`"json"`))
+ if !strings.HasPrefix(grammar, "root ::= ollama-thinking-0\n") || !strings.Contains(grammar, "ollama-format ::= object\n") {
+ t.Errorf("grammar does not wrap the json grammar:\n%s", grammar)
+ }
+ if got := conversions.Load(); got != 1 {
+ t.Errorf("json format converted a schema, %d conversions", got)
+ }
+}