Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
llm: apply a format after the thinking in one llama-server request
Jesse Gross2ff052b78108
diff --git a/llm/gbnf.go b/llm/gbnf.gonew file mode 100644index 000000000..8703386b2--- /dev/null+++ b/llm/gbnf.go@@ -0,0 +1,180 @@+package llm++import (+ "fmt"+ "slices"+ "strings"+ "unicode"+)++// thinkingGrammar returns a grammar that leaves the text before any of the+// closings unconstrained, then constrains what follows the first complete+// closing to the root of the format grammar. The response may end before any+// closing. A closing that is a prefix of another ends the thinking as soon+// as it is complete.+func thinkingGrammar(closings []string, format string) string {+ // The rules added here take a prefix no rule of the format grammar starts+ // with.+ prefix := "ollama-"+ for grammarHasRulePrefix(format, prefix) {+ prefix += "ollama-"+ }+ thinking, formatted := prefix+"thinking-", prefix+"format"++ a := newClosingAutomaton(closings)+ var b strings.Builder+ fmt.Fprintf(&b, "root ::= %s0\n", thinking)+ for state := range a.states {+ fmt.Fprintf(&b, "%s%d ::=", thinking, state)+ var matching, terminal []rune+ next := map[int][]rune{}+ for _, r := range a.alphabet {+ to, done := a.next(state, r)+ if done {+ terminal = append(terminal, r)+ } else if to != 0 {+ next[to] = append(next[to], r)+ }+ if done || to != 0 {+ matching = append(matching, r)+ }+ }+ // Any other character restarts the search from the empty prefix.+ fmt.Fprintf(&b, " | %s %s0", gbnfCharClass(matching, true), thinking)+ for to := range a.states {+ if rs := next[to]; len(rs) > 0 {+ fmt.Fprintf(&b, " | %s %s%d", gbnfCharClass(rs, false), thinking, to)+ }+ }+ if len(terminal) > 0 {+ fmt.Fprintf(&b, " | %s %s", gbnfCharClass(terminal, false), formatted)+ }+ b.WriteByte('\n')+ }+ b.WriteString(renameGrammarRule(format, "root", formatted))+ return b.String()+}++// closingAutomaton is the Aho-Corasick automaton over the closings: each+// state is a proper prefix of a closing, state 0 the empty one.+type closingAutomaton struct {+ closings []string+ states []string+ index map[string]int+ alphabet []rune+}++func newClosingAutomaton(closings []string) *closingAutomaton {+ a := &closingAutomaton{closings: closings, index: map[string]int{}}+ add := func(prefix string) {+ if _, ok := a.index[prefix]; !ok {+ a.index[prefix] = len(a.states)+ a.states = append(a.states, prefix)+ }+ }+ add("")+ for _, closing := range closings {+ for i := range closing {+ add(closing[:i])+ }+ for _, r := range closing {+ if !slices.Contains(a.alphabet, r) {+ a.alphabet = append(a.alphabet, r)+ }+ }+ }+ slices.Sort(a.alphabet)+ return a+}++// next returns the state after reading r in state, or done when the text+// read so far ends with a closing.+func (a *closingAutomaton) next(state int, r rune) (to int, done bool) {+ s := a.states[state] + string(r)+ for _, closing := range a.closings {+ if strings.HasSuffix(s, closing) {+ return 0, true+ }+ }+ for i := range s {+ if to, ok := a.index[s[i:]]; ok {+ return to, false+ }+ }+ return 0, false+}++func gbnfCharClass(rs []rune, negate bool) string {+ var b strings.Builder+ b.WriteByte('[')+ if negate {+ b.WriteByte('^')+ }+ for _, r := range rs {+ switch {+ case r > unicode.MaxASCII || !unicode.IsPrint(r) || strings.ContainsRune(`]\^-`, r):+ fmt.Fprintf(&b, `\U%08X`, r)+ default:+ b.WriteRune(r)+ }+ }+ b.WriteByte(']')+ return b.String()+}++func renameGrammarRule(grammar, from, to string) string {+ var b strings.Builder+ scanGrammar(grammar, func(text string, name bool) {+ if name && text == from {+ text = to+ }+ b.WriteString(text)+ })+ return b.String()+}++func grammarHasRulePrefix(grammar, prefix string) bool {+ found := false+ scanGrammar(grammar, func(text string, name bool) {+ found = found || name && strings.HasPrefix(text, prefix)+ })+ return found+}++// scanGrammar splits a grammar into rule names and everything else. Comments,+// quoted strings and character classes are passed whole, so a word inside+// them is never taken for a name.+func scanGrammar(grammar string, fn func(text string, name bool)) {+ isName := func(c byte) bool {+ return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-' || c == '_'+ }+ for i := 0; i < len(grammar); {+ start := i+ switch c := grammar[i]; {+ case c == '#':+ for i < len(grammar) && grammar[i] != '\n' {+ i+++ }+ case c == '"' || c == '[':+ end := byte('"')+ if c == '[' {+ end = ']'+ }+ for i++; i < len(grammar) && grammar[i] != end; i++ {+ if grammar[i] == '\\' {+ i+++ }+ }+ i = min(i+1, len(grammar))+ case isName(c):+ for i < len(grammar) && isName(grammar[i]) {+ i+++ }+ fn(grammar[start:i], true)+ continue+ default:+ i+++ }+ fn(grammar[start:i], false)+ }+}diff --git a/llm/gbnf_test.go b/llm/gbnf_test.gonew file mode 100644index 000000000..4ae1456f9--- /dev/null+++ b/llm/gbnf_test.go@@ -0,0 +1,79 @@+package llm++import (+ "strings"+ "testing"+)++// TestRenameGrammarRule checks that only rule names change, not the same word+// in a comment, a string, a character class or a longer name.+func TestRenameGrammarRule(t *testing.T) {+ grammar := "# root stays in a comment\n" ++ "root ::= \"root\" | [root] | \"[\" root \"]\" | root-item\n" ++ "root-item ::= \"\\\"root\\\"\" root\n"+ want := "# root stays in a comment\n" ++ "out ::= \"root\" | [root] | \"[\" out \"]\" | root-item\n" ++ "root-item ::= \"\\\"root\\\"\" out\n"+ if got := renameGrammarRule(grammar, "root", "out"); got != want {+ t.Errorf("renamed grammar:\n%s\nwant:\n%s", got, want)+ }+}++// TestClosingAutomatonNext checks that the state after a character is the+// longest suffix of the text so far that is a proper prefix of a closing, and+// that the automaton is done exactly when the text ends with a closing. The+// closings share a prefix and overlap, so a reset to the empty prefix on a+// mismatch would lose matches.+func TestClosingAutomatonNext(t *testing.T) {+ a := newClosingAutomaton([]string{"<|x|>", "<|y|>", "y|>|"})+ state := 0+ for _, step := range []struct {+ r rune+ want string+ done bool+ }{+ {r: '<', want: "<"},+ {r: '|', want: "<|"},+ {r: '<', want: "<"},+ {r: '|', want: "<|"},+ {r: 'y', want: "<|y"},+ {r: '|', want: "<|y|"},+ {r: '>', done: true},+ {r: '|', want: ""},+ {r: 'y', want: "y"},+ {r: '|', want: "y|"},+ {r: '>', want: "y|>"},+ {r: '<', want: "<"},+ {r: 'z', want: ""},+ } {+ to, done := a.next(state, step.r)+ if done != step.done || (!done && a.states[to] != step.want) {+ t.Fatalf("after %q: state %q done %v, want state %q done %v", step.r, a.states[to], done, step.want, step.done)+ }+ state = to+ }+}++// TestThinkingGrammarRules checks the emitted rules: every prefix state may+// end the response, completing the closing leads to the renamed format root,+// the format grammar's other rules keep their names, and the added rules+// avoid a prefix the format grammar already uses.+func TestThinkingGrammarRules(t *testing.T) {+ format := "root ::= \"{\" space \"}\"\nspace ::= | \" \"\n"+ grammar := thinkingGrammar([]string{"</think>"}, format)+ for _, want := range []string{+ "root ::= ollama-thinking-0\n",+ "ollama-thinking-0 ::= | [^<] ollama-thinking-0 | [<] ollama-thinking-1\n",+ "ollama-thinking-7 ::= | [^<>] ollama-thinking-0 | [<] ollama-thinking-1 | [>] ollama-format\n",+ "ollama-format ::= \"{\" space \"}\"\nspace ::= | \" \"\n",+ } {+ if !strings.Contains(grammar, want) {+ t.Errorf("grammar lacks %q:\n%s", want, grammar)+ }+ }++ grammar = thinkingGrammar([]string{"</think>"}, "root ::= ollama-x\nollama-x ::= \"{}\"\n")+ if !strings.HasPrefix(grammar, "root ::= ollama-ollama-thinking-0\n") || !strings.Contains(grammar, "ollama-ollama-format ::= ollama-x\n") {+ t.Errorf("grammar does not avoid the format's rule prefix:\n%s", grammar)+ }+}diff --git a/llm/llama_server.go b/llm/llama_server.goindex 7a0b1ed33..c4eb4adb0 100644--- a/llm/llama_server.go+++ b/llm/llama_server.go@@ -5,9 +5,10 @@// still render prompts in Go and call /completion. Other GGUF chat models use// llama-server's chat_template handling through /v1/chat/completions.//-// For structured output, JSON schemas are passed directly to llama-server via-// its json_schema field (avoiding the CGO SchemaToGrammar dependency). Raw BNF-// grammars are passed via the grammar field.+// For structured output, a JSON schema is passed to llama-server via its+// json_schema field and the "json" format as a builtin grammar via the grammar+// field. A format that applies after a response's thinking is sent as a+// grammar built around the GBNF llama-server itself derives from the schema.//// llama-server auto-detects GPU layers (-ngl), thread count (-t), and flash// attention (--flash-attn).@@ -16,6 +17,7 @@ package llmimport ("bufio""bytes"+ "container/list""context"crand "crypto/rand""encoding/base64"@@ -1425,6 +1427,85 @@ type llamaServerCompletionRequest struct {TimingsPerToken bool `json:"timings_per_token,omitempty"`}+// schemaGrammars caches the grammars llama-server derives from JSON schemas,+// most recently used first. The conversion depends only on the llama-server+// build, which is fixed for the process, so entries are shared by every+// runner and never expire.+var schemaGrammars = struct {+ sync.Mutex+ entries map[string]*list.Element+ order list.List+}{entries: map[string]*list.Element{}}++const schemaGrammarsSize = 64++type schemaGrammar struct {+ schema, grammar string+}++// schemaGrammar converts a JSON schema to GBNF with llama-server's own+// converter. An empty completion evaluates and generates nothing, but its+// final response still reports the grammar the schema was converted to.+func (s *llamaServerRunner) schemaGrammar(ctx context.Context, schema json.RawMessage) (string, error) {+ key := string(schema)+ schemaGrammars.Lock()+ if e, ok := schemaGrammars.entries[key]; ok {+ schemaGrammars.order.MoveToFront(e)+ schemaGrammars.Unlock()+ return e.Value.(*schemaGrammar).grammar, nil+ }+ schemaGrammars.Unlock()++ body, err := json.Marshal(struct {+ Prompt [][]int `json:"prompt"`+ NPredict int `json:"n_predict"`+ JsonSchema json.RawMessage `json:"json_schema"`+ ResponseFields []string `json:"response_fields"`+ }{Prompt: [][]int{{}}, JsonSchema: schema, ResponseFields: []string{"generation_settings/grammar"}})+ if err != nil {+ return "", fmt.Errorf("failed to marshal grammar request: %v", err)+ }+ endpoint := fmt.Sprintf("http://127.0.0.1:%d/completion", s.port)+ serverReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))+ if err != nil {+ return "", fmt.Errorf("error creating grammar request: %v", err)+ }+ serverReq.Header.Set("Content-Type", "application/json")+ res, err := s.httpClient().Do(serverReq)+ if err != nil {+ return "", fmt.Errorf("llama-server grammar request failed: %v", err)+ }+ defer res.Body.Close()+ resBody, err := io.ReadAll(res.Body)+ if err != nil {+ return "", fmt.Errorf("failed reading llama-server grammar response: %w", err)+ }+ if res.StatusCode >= 400 {+ return "", api.StatusError{StatusCode: res.StatusCode, ErrorMessage: s.statusErrorMessage(resBody)}+ }+ var lsRes struct {+ Grammar string `json:"generation_settings/grammar"`+ }+ if err := json.Unmarshal(resBody, &lsRes); err != nil {+ return "", fmt.Errorf("error unmarshalling llama-server grammar response: %v", err)+ }+ if lsRes.Grammar == "" {+ return "", errors.New("llama-server returned no grammar for the schema")+ }++ schemaGrammars.Lock()+ defer schemaGrammars.Unlock()+ if _, ok := schemaGrammars.entries[key]; !ok {+ if schemaGrammars.order.Len() == schemaGrammarsSize {+ oldest := schemaGrammars.order.Back()+ delete(schemaGrammars.entries, oldest.Value.(*schemaGrammar).schema)+ schemaGrammars.order.Remove(oldest)+ }+ schemaGrammars.entries[key] = schemaGrammars.order.PushFront(&schemaGrammar{schema: key, grammar: lsRes.Grammar})+ }+ return lsRes.Grammar, nil+}+func llamaServerPreservedTokens(parserTokens []string, toolCallTag string) []string {tokens := append([]string{}, parserTokens...)tokens = append(tokens, llamaServerPreservedTokensForToolTag(toolCallTag)...)@@ -1611,6 +1692,19 @@ func (s *llamaServerRunner) Completion(ctx context.Context, req CompletionReques}}+ // A format on a thinking response applies after the closing string, which+ // only a grammar of our own can express, so a schema is converted first.+ if len(req.ThinkingClose) > 0 && (lsReq.Grammar != "" || lsReq.JsonSchema != nil) {+ if lsReq.Grammar == "" {+ grammar, err := s.schemaGrammar(ctx, lsReq.JsonSchema)+ if err != nil {+ return err+ }+ lsReq.Grammar, lsReq.JsonSchema = grammar, nil+ }+ lsReq.Grammar = thinkingGrammar(req.ThinkingClose, lsReq.Grammar)+ }+// Convert media: replace Ollama's stable [img-N] markers with the per-process// llama-server media marker and package the matching payloads as base64.if len(req.Media) > 0 {diff --git a/llm/llama_server_test.go b/llm/llama_server_test.goindex 60193ff9c..7c761b348 100644--- a/llm/llama_server_test.go+++ b/llm/llama_server_test.go@@ -3847,3 +3847,116 @@ func fakeRunningCmd() *exec.Cmd {// SIGKILL children when the test process exits.return cmd}++// TestLlamaServerCompletionThinkingFormat checks that a format on a thinking+// response is one request carrying a grammar: a schema is converted by an+// empty completion once per schema, "json" needs no conversion, the grammar+// wraps the format rules behind the closing string, and content and metrics+// pass through unchanged.+func TestLlamaServerCompletionThinkingFormat(t *testing.T) {+ schema := json.RawMessage(`{"type":"object"}`)+ converted := "root ::= \"{\" space \"}\"\nspace ::= | \" \"\n"+ sseLines := []string{+ `data: {"content":"Let me think.</think>","stop":false,"timings":{"cache_n":2,"prompt_n":3,"prompt_ms":10,"predicted_n":3,"predicted_ms":15}}`,+ `data: {"content":"{}","stop":false,"timings":{"cache_n":2,"prompt_n":3,"prompt_ms":10,"predicted_n":4,"predicted_ms":20}}`,+ `data: {"content":"","stop":true,"stop_type":"eos","timings":{"cache_n":2,"prompt_n":3,"prompt_ms":10,"predicted_n":4,"predicted_ms":20}}`,+ }++ var conversions atomic.Int32+ grammars := make(chan string, 8)+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path == "/health" {+ fmt.Fprint(w, `{"status":"ok"}`)+ return+ }+ var reqBody struct {+ Prompt any `json:"prompt"`+ NPredict *int `json:"n_predict"`+ JsonSchema json.RawMessage `json:"json_schema"`+ Grammar string `json:"grammar"`+ ResponseFields []string `json:"response_fields"`+ }+ if err := json.NewDecoder(r.Body).Decode(&reqBody); err != nil {+ t.Errorf("invalid request body: %v", err)+ return+ }+ if len(reqBody.ResponseFields) > 0 {+ conversions.Add(1)+ if !reflect.DeepEqual(reqBody.Prompt, []any{[]any{}}) || reqBody.NPredict == nil || *reqBody.NPredict != 0 {+ t.Errorf("conversion request prompt %v n_predict %v, want an empty token prompt and no generation", reqBody.Prompt, reqBody.NPredict)+ }+ if !bytes.Equal(reqBody.JsonSchema, schema) || !reflect.DeepEqual(reqBody.ResponseFields, []string{"generation_settings/grammar"}) {+ t.Errorf("conversion request schema %s fields %v", reqBody.JsonSchema, reqBody.ResponseFields)+ }+ json.NewEncoder(w).Encode(map[string]string{"generation_settings/grammar": converted})+ return+ }+ if reqBody.JsonSchema != nil {+ t.Errorf("completion carried schema %s", reqBody.JsonSchema)+ }+ grammars <- reqBody.Grammar+ w.Header().Set("Content-Type", "text/event-stream")+ for _, line := range sseLines {+ fmt.Fprintln(w, line)+ fmt.Fprintln(w)+ }+ }))+ defer srv.Close()++ parts := strings.Split(srv.URL, ":")+ var portInt int+ fmt.Sscanf(parts[len(parts)-1], "%d", &portInt)++ runner := &llamaServerRunner{+ port: portInt,+ cmd: fakeRunningCmd(),+ sem: semaphore.NewWeighted(1),+ options: api.Options{Runner: api.Runner{NumCtx: 2048}},+ }++ complete := func(format json.RawMessage) string {+ var responses []CompletionResponse+ opts := api.DefaultOptions()+ err := runner.Completion(t.Context(), CompletionRequest{+ Prompt: "test prompt",+ Format: format,+ ThinkingClose: []string{"</think>"},+ Options: &opts,+ }, func(cr CompletionResponse) {+ responses = append(responses, cr)+ })+ if err != nil {+ t.Fatalf("Completion error: %v", err)+ }+ var content strings.Builder+ for _, resp := range responses[:len(responses)-1] {+ content.WriteString(resp.Content)+ }+ if got := content.String(); got != "Let me think.</think>{}" {+ t.Errorf("streamed content = %q", got)+ }+ final := responses[len(responses)-1]+ if !final.Done || final.DoneReason != DoneReasonStop || final.EvalCount != 4 || final.PromptEvalCount != 5 {+ t.Errorf("final response = %+v, want done with 4 generated and 5 prompt tokens", final)+ }+ return <-grammars+ }++ for range 2 {+ grammar := complete(schema)+ if !strings.HasPrefix(grammar, "root ::= ollama-thinking-0\n") || !strings.Contains(grammar, "ollama-format ::= \"{\" space \"}\"\n") {+ t.Errorf("grammar does not wrap the converted schema:\n%s", grammar)+ }+ }+ if got := conversions.Load(); got != 1 {+ t.Errorf("schema converted %d times, want once", got)+ }++ grammar := complete(json.RawMessage(`"json"`))+ if !strings.HasPrefix(grammar, "root ::= ollama-thinking-0\n") || !strings.Contains(grammar, "ollama-format ::= object\n") {+ t.Errorf("grammar does not wrap the json grammar:\n%s", grammar)+ }+ if got := conversions.Load(); got != 1 {+ t.Errorf("json format converted a schema, %d conversions", got)+ }+}