Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
feat: add System One scoring API (#18606)
Parth Sareenffb6f220d0d8
diff --git a/decision/systemone.go b/decision/systemone.gonew file mode 100644index 000000000..8f07da7e2--- /dev/null+++ b/decision/systemone.go@@ -0,0 +1,228 @@+// Package decision compiles typed decision requests into candidate-scoring+// prompts and converts the resulting logits into /v1/systemone responses.+package decision++import (+ "bytes"+ "encoding/json"+ "fmt"+ "math"+ "slices"+ "strconv"+ "strings"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/internal/orderedmap"+ "github.com/ollama/ollama/llm"+)++type compiledField struct {+ Field+ typ string+}++type Compiled struct {+ Request llm.ScoreRequest+ fields []compiledField+ messages [][]api.Message+}++func Compile(req Request) (*Compiled, error) {+ if strings.TrimSpace(req.Model) == "" {+ return nil, fmt.Errorf("model is required")+ }+ if req.Questions.Len() < 1 || req.Questions.Len() > 64 {+ return nil, fmt.Errorf("questions must contain 1–64 fields")+ }+ context, err := content(req.State)+ if err != nil {+ return nil, fmt.Errorf("state: %w", err)+ }+ if strings.TrimSpace(context) == "" {+ return nil, fmt.Errorf("state must not be empty")+ }+ c := &Compiled{}+ for name, q := range req.Questions.All() {+ f, err := compileField(name, q)+ if err != nil {+ return nil, fmt.Errorf("question %q: %w", name, err)+ }+ c.fields = append(c.fields, f)+ }+ data, err := json.Marshal(struct {+ Context string `json:"context"`+ Schema []compiledField `json:"schema"`+ }{context, c.fields})+ if err != nil {+ return nil, err+ }+ for _, f := range c.fields {+ name, err := json.Marshal(f.Name)+ if err != nil {+ return nil, err+ }+ c.messages = append(c.messages, []api.Message{+ {Role: "user", Content: string(data) + "\n\nRequested field: " + string(name)},+ })+ var row llm.ScoreRow+ for _, choice := range f.Choices {+ row.Candidates = append(row.Candidates, choice.Code)+ }+ c.Request.Rows = append(c.Request.Rows, row)+ }+ return c, nil+}++// Render prepares scoring prompts using the model's system prompt and chat template.+// The caller must set Request.MaxTokens to the loaded context size before scoring.+func (c *Compiled) Render(render func([]api.Message) (string, error)) error {+ for i, messages := range c.messages {+ prompt, err := render(messages)+ if err != nil {+ return err+ }+ c.Request.Rows[i].Prompt = prompt+ }+ return nil+}++func content(raw json.RawMessage) (string, error) {+ raw = bytes.TrimSpace(raw)+ if len(raw) == 0 {+ return "", fmt.Errorf("must be a string, object, or array")+ }+ switch raw[0] {+ case '"':+ var s string+ err := json.Unmarshal(raw, &s)+ return s, err+ case '{', '[':+ var b bytes.Buffer+ err := json.Compact(&b, raw)+ return b.String(), err+ default:+ return "", fmt.Errorf("must be a string, object, or array")+ }+}++func compileField(name string, q Question) (compiledField, error) {+ f := compiledField{Field: Field{Name: name}, typ: q.Type}+ if strings.TrimSpace(name) == "" {+ return f, fmt.Errorf("field name must not be empty")+ }+ description, err := content(q.Instructions)+ if err != nil || strings.TrimSpace(description) == "" {+ return f, fmt.Errorf("instructions must be a nonempty string, object, or array")+ }+ f.Description = description+ add := func(value any, description string) {+ f.Choices = append(f.Choices, Choice{string(rune('A' + len(f.Choices))), value, description})+ }+ switch q.Type {+ case "noul":+ criteria := orderedmap.New[string, *string]()+ if len(q.Criteria) > 0 {+ if err := json.Unmarshal(q.Criteria, criteria); err != nil || string(q.Criteria) == "null" {+ return f, fmt.Errorf("noul criteria must be an object of true/false descriptions")+ }+ }+ no, yes := "No", "Yes"+ for key, value := range criteria.All() {+ if value == nil {+ return f, fmt.Errorf("noul descriptions must be strings")+ }+ switch key {+ case "false":+ no = *value+ case "true":+ yes = *value+ default:+ return f, fmt.Errorf("unknown noul criterion %q", key)+ }+ }+ add(false, no)+ add(true, yes)+ case "choice":+ criteria := orderedmap.New[string, *string]()+ if err := json.Unmarshal(q.Criteria, criteria); err != nil {+ return f, fmt.Errorf("choice criteria must map option keys to descriptions or null")+ }+ for key, value := range criteria.All() {+ if strings.TrimSpace(key) == "" {+ return f, fmt.Errorf("choice keys must not be empty")+ }+ description := key+ if value != nil {+ description = *value+ }+ add(key, description)+ }+ case "score":+ var criteria []*string+ if err := json.Unmarshal(q.Criteria, &criteria); err != nil {+ return f, fmt.Errorf("score criteria must be an array of descriptions")+ }+ for i, description := range criteria {+ if description == nil {+ return f, fmt.Errorf("score descriptions must be strings")+ }+ add(strconv.Itoa(i), *description)+ }+ default:+ return f, fmt.Errorf("type must be choice, noul, or score")+ }+ if len(f.Choices) < 2 || len(f.Choices) > 26 {+ return f, fmt.Errorf("criteria must contain 2–26 candidates")+ }+ return f, nil+}++func (c *Compiled) Answer(model string, result llm.ScoreResponse) (Response, error) {+ response := Response{Model: model, Answers: &Answers{}, Usage: Usage{InputTokens: result.InputTokens, OutputTokens: result.OutputTokens}}+ if len(result.Logits) != len(c.fields) {+ return response, fmt.Errorf("scorer returned %d rows for %d questions", len(result.Logits), len(c.fields))+ }+ for i, f := range c.fields {+ logits := result.Logits[i]+ if len(logits) != len(f.Choices) {+ return response, fmt.Errorf("scorer returned the wrong number of candidates for %q", f.Name)+ }+ peak := slices.Max(logits)+ p := make([]float64, len(logits))+ var sum float64+ for j, logit := range logits {+ if math.IsNaN(float64(logit)) || math.IsInf(float64(logit), 0) {+ return response, fmt.Errorf("scorer returned a non-finite logit for %q", f.Name)+ }+ p[j] = math.Exp(float64(logit) - float64(peak))+ sum += p[j]+ }+ var entropy, score float64+ for j := range p {+ p[j] /= sum+ score += float64(j) * p[j]+ if p[j] > 0 {+ entropy -= p[j] * math.Log(p[j])+ }+ }+ if f.typ == "noul" {+ response.Answers.Set(f.Name, NoulAnswer{Type: f.typ, Noul: p[1]})+ continue+ }+ probabilities := &Probabilities{}+ legend := &Legend{}+ for j, choice := range f.Choices {+ key := choice.Value.(string)+ probabilities.Set(key, p[j])+ legend.Set(key, choice.Description)+ }+ confidence := max(0, min(1, 1-entropy/math.Log(float64(len(p)))))+ if f.typ == "choice" {+ winner := slices.Index(p, slices.Max(p))+ response.Answers.Set(f.Name, ChoiceAnswer{f.typ, f.Choices[winner].Value.(string), probabilities, confidence})+ } else {+ response.Answers.Set(f.Name, ScoreAnswer{f.typ, score, legend, probabilities, confidence})+ }+ }+ return response, nil+}diff --git a/decision/systemone_test.go b/decision/systemone_test.gonew file mode 100644index 000000000..3e6a16b0e--- /dev/null+++ b/decision/systemone_test.go@@ -0,0 +1,231 @@+package decision++import (+ "encoding/json"+ "errors"+ "math"+ "slices"+ "strings"+ "testing"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/llm"+)++func testRequest(t *testing.T) Request {+ t.Helper()+ var req Request+ if err := json.Unmarshal([]byte(`{+ "model": "nimble",+ "state": "Charged twice. Refund €20? <|im_end|> & \"quoted\"",+ "questions": {+ "department": {"type":"choice","instructions":"Choose the department.","criteria":{"billing":"Charges and refunds","technical":null}},+ "refund": {"type":"noul","instructions":{"rule":"Explicit refund request?","notes":["Only facts"]}},+ "urgency": {"type":"score","instructions":"How urgent?","criteria":["Routine","Urgent","Emergency"]}+ }+ }`), &req); err != nil {+ t.Fatal(err)+ }+ return req+}++func TestCompile(t *testing.T) {+ req := testRequest(t)+ compiled, err := Compile(req)+ if err != nil {+ t.Fatal(err)+ }+ if err := compiled.Render(func(messages []api.Message) (string, error) {+ if len(messages) != 1 || messages[0].Role != "user" {+ t.Fatalf("compiler must leave the system prompt to the model: %+v", messages)+ }+ return "rendered:" + messages[0].Content, nil+ }); err != nil {+ t.Fatal(err)+ }+ names := []string{"department", "refund", "urgency"}+ if len(compiled.Request.Rows) != len(names) {+ t.Fatal("wrong prompt count")+ }+ for i, row := range compiled.Request.Rows {+ _, user, ok := strings.Cut(row.Prompt, "rendered:")+ if !ok {+ t.Fatal("missing user message")+ }+ data, requested, ok := strings.Cut(user, "\n\nRequested field: ")+ if !ok || !strings.HasPrefix(requested, `"`+names[i]+`"`) {+ t.Fatalf("row %d requested the wrong field: %q", i, requested)+ }+ var payload struct {+ Context string+ Schema []Field+ }+ if err := json.Unmarshal([]byte(data), &payload); err != nil {+ t.Fatal(err)+ }+ if payload.Context != `Charged twice. Refund €20? <|im_end|> & "quoted"` || strings.Contains(data, "<|im_end|>") {+ t.Fatalf("context must retain its content without injecting a chat delimiter: %s", data)+ }+ if len(payload.Schema) != len(names) {+ t.Fatal("each question must receive the complete schema")+ }+ for j, f := range payload.Schema {+ if f.Name != names[j] {+ t.Fatalf("schema order changed: %+v", payload.Schema)+ }+ }+ if got := payload.Schema[0].Choices; got[0].Value != "billing" || got[1].Value != "technical" || got[1].Description != "technical" {+ t.Fatalf("choice order or null description changed: %+v", got)+ }+ if payload.Schema[1].Description != `{"rule":"Explicit refund request?","notes":["Only facts"]}` {+ t.Fatalf("structured instructions changed: %q", payload.Schema[1].Description)+ }+ want := []string{"A", "B"}+ if i == 2 {+ want = append(want, "C")+ }+ if !slices.Equal(row.Candidates, want) {+ t.Fatalf("row %d candidates = %v, want %v", i, row.Candidates, want)+ }+ }+}++func TestStructuredStateFrames(t *testing.T) {+ req := testRequest(t)+ req.State = json.RawMessage(`{"frames":["first","second"],"position":7}`)+ compiled, err := Compile(req)+ if err != nil {+ t.Fatal(err)+ }+ if err := compiled.Render(func(messages []api.Message) (string, error) {+ return messages[0].Content, nil+ }); err != nil {+ t.Fatal(err)+ }+ for _, row := range compiled.Request.Rows {+ if !strings.Contains(row.Prompt, `"context":"{\"frames\":[\"first\",\"second\"],\"position\":7}"`) {+ t.Fatalf("structured state was not preserved in the text prompt: %q", row.Prompt)+ }+ }+}++func TestRenderError(t *testing.T) {+ compiled, err := Compile(testRequest(t))+ if err != nil {+ t.Fatal(err)+ }+ want := errors.New("invalid model template")+ if err := compiled.Render(func([]api.Message) (string, error) { return "", want }); !errors.Is(err, want) {+ t.Fatalf("Render error = %v, want %v", err, want)+ }+}++func TestAnswers(t *testing.T) {+ req := testRequest(t)+ c, err := Compile(req)+ if err != nil {+ t.Fatal(err)+ }+ result, err := c.Answer("nimble", llm.ScoreResponse{+ Logits: [][]float32{{0, 0}, {-1000, 1000}, {1000, 1000, 1000}}, InputTokens: 900,+ })+ if err != nil {+ t.Fatal(err)+ }+ choice, _ := result.Answers.Get(c.fields[0].Name)+ got := choice.(ChoiceAnswer)+ if got.Choice != "billing" || got.Confidence != 0 {+ t.Fatalf("tie must choose the first candidate with zero concentration: %+v", got)+ }+ noul, _ := result.Answers.Get("refund")+ if noul.(NoulAnswer).Noul != 1 {+ t.Fatal("noul must report P(true), using stable softmax")+ }+ score, _ := result.Answers.Get("urgency")+ if math.Abs(score.(ScoreAnswer).Score-1) > 1e-12 || score.(ScoreAnswer).Legend.Len() != 3 {+ t.Fatalf("bad expected rubric index or legend: %+v", score)+ }+ if result.Usage.InputTokens != 900 || result.Usage.OutputTokens != 0 {+ t.Fatalf("bad direct-scoring usage: %+v", result.Usage)+ }+ for _, logits := range [][][]float32{nil, {{1}, {1, 2}, {1, 2, 3}}, {{float32(math.NaN()), 0}, {0, 1}, {1, 2, 3}}} {+ if _, err := c.Answer("nimble", llm.ScoreResponse{Logits: logits}); err == nil {+ t.Errorf("accepted malformed runner result: %v", logits)+ }+ }+}++func TestAnswerLogprobs(t *testing.T) {+ c, err := Compile(testRequest(t))+ if err != nil {+ t.Fatal(err)+ }+ logits := [][]float32{{0, 2}, {0, -2}, {0, 2, 4}}+ logprobs := [][]float32{{-10, -8}, {-10, -12}, {-10, -8, -6}}+ want, err := c.Answer("nimble", llm.ScoreResponse{Logits: logits, InputTokens: 123})+ if err != nil {+ t.Fatal(err)+ }+ got, err := c.Answer("nimble", llm.ScoreResponse{Logits: logprobs, InputTokens: 123, OutputTokens: 7})+ if err != nil {+ t.Fatal(err)+ }+ wantAnswers, _ := json.Marshal(want.Answers)+ gotAnswers, _ := json.Marshal(got.Answers)+ if string(wantAnswers) != string(gotAnswers) {+ t.Fatalf("row offsets changed candidate probabilities: %s != %s", gotAnswers, wantAnswers)+ }+ if got.Usage.InputTokens != 123 || got.Usage.OutputTokens != 7 || want.Usage.OutputTokens != 0 {+ t.Fatalf("incorrect backend usage: llama=%+v direct=%+v", got.Usage, want.Usage)+ }+}++func TestInvalidRequests(t *testing.T) {+ for _, data := range []string{+ `{}`, `{"model":"nimble","state":"x","questions":{}}`,+ `{"model":"nimble","state":" ","questions":{"x":{"type":"noul","instructions":"q"}}}`,+ `{"model":"nimble","state":null,"questions":{"x":{"type":"noul","instructions":"q"}}}`,+ `{"model":"nimble","state":"x","questions":{"x":{"type":"other","instructions":"q"}}}`,+ `{"model":"nimble","state":"x","questions":{"x":{"type":"noul"}}}`,+ `{"model":"nimble","state":"x","questions":{"":{"type":"noul","instructions":"q"}}}`,+ `{"model":"nimble","state":"x","questions":{"x":{"type":"choice","instructions":"q","criteria":{"a":"a"}}}}`,+ `{"model":"nimble","state":"x","questions":{"x":{"type":"score","instructions":"q","criteria":["a",null]}}}`,+ `{"model":"nimble","state":"x","questions":{"x":{"type":"noul","instructions":"q","criteria":{"true":null}}}}`,+ `{"model":"nimble","state":"x","questions":{"x":{"type":"noul","instructions":"q","criteria":{"yes":"Yes"}}}}`,+ } {+ t.Run(data, func(t *testing.T) {+ var req Request+ if err := json.Unmarshal([]byte(data), &req); err != nil {+ t.Fatal(err)+ }+ if _, err := Compile(req); err == nil {+ t.Fatal("accepted invalid request")+ }+ })+ }+ req := testRequest(t)+ q, _ := req.Questions.Get("urgency")+ q.Criteria = json.RawMessage(`["x"` + strings.Repeat(`,"x"`, 26) + `]`)+ req.Questions.Set("urgency", q)+ if _, err := Compile(req); err == nil {+ t.Fatal("accepted 27 candidates")+ }+}++func TestContent(t *testing.T) {+ for _, tt := range []struct{ input, want string }{+ {` "line\nquoted \"text\"" `, "line\nquoted \"text\""},+ {` {"z": 9007199254740993, "a": 1e999} `, `{"z":9007199254740993,"a":1e999}`},+ {` [1.0, 1e6, -0, "€"] `, `[1.0,1e6,-0,"€"]`},+ } {+ got, err := content(json.RawMessage(tt.input))+ if err != nil || got != tt.want {+ t.Errorf("content(%s) = %q, %v; want %q", tt.input, got, err, tt.want)+ }+ }+ for _, input := range []string{"", "null", "true", "42", `"unterminated`, `{"x":}`, `[1,]`} {+ if _, err := content(json.RawMessage(input)); err == nil {+ t.Errorf("accepted invalid content %q", input)+ }+ }+}diff --git a/decision/types.go b/decision/types.gonew file mode 100644index 000000000..a3dfb6481--- /dev/null+++ b/decision/types.go@@ -0,0 +1,76 @@+package decision++import (+ "encoding/json"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/internal/orderedmap"+)++// Questions preserves request order, which determines the order of scored fields.+// Its zero value is ready to use with Set.+type Questions = orderedmap.Map[string, Question]++type Request struct {+ Model string `json:"model"`+ State json.RawMessage `json:"state"`+ Questions *Questions `json:"questions"`+ KeepAlive *api.Duration `json:"keep_alive,omitempty"`+}++type Question struct {+ Type string `json:"type"`+ Instructions json.RawMessage `json:"instructions"`+ Criteria json.RawMessage `json:"criteria"`+}++// Choice maps a single-token answer code to a value in the question's schema.+type Choice struct {+ Code string `json:"code"`+ Value any `json:"value"`+ Description string `json:"description"`+}++// Field is a question's schema as presented to the model.+type Field struct {+ Name string `json:"name"`+ Description string `json:"description"`+ Choices []Choice `json:"choices"`+}++type (+ Answers = orderedmap.Map[string, any]+ Probabilities = orderedmap.Map[string, float64]+ Legend = orderedmap.Map[string, string]+)++type Response struct {+ Model string `json:"model"`+ Answers *Answers `json:"answers"`+ Usage Usage `json:"usage"`+}++type Usage struct {+ InputTokens int `json:"input_tokens"`+ OutputTokens int `json:"output_tokens"`+}++type NoulAnswer struct {+ Type string `json:"type"`+ Noul float64 `json:"noul"`+}++type ChoiceAnswer struct {+ Type string `json:"type"`+ Choice string `json:"choice"`+ Probabilities *Probabilities `json:"probabilities"`+ Confidence float64 `json:"confidence"`+}++type ScoreAnswer struct {+ Type string `json:"type"`+ Score float64 `json:"score"`+ Legend *Legend `json:"legend"`+ Probabilities *Probabilities `json:"probabilities"`+ Confidence float64 `json:"confidence"`+}diff --git a/llm/llama_server_score.go b/llm/llama_server_score.gonew file mode 100644index 000000000..b5d64a292--- /dev/null+++ b/llm/llama_server_score.go@@ -0,0 +1,281 @@+package llm++import (+ "bytes"+ "context"+ "encoding/json"+ "fmt"+ "io"+ "math"+ "net/http"+ "slices"++ "github.com/ollama/ollama/api"+)++var _ Scorer = (*llamaServerRunner)(nil)++const (+ // Candidates are lifted above every other token so that top_k keeps exactly+ // them; a shared bias does not change their softmax. A larger bias is+ // retried if another token still outranks a candidate.+ scoreBias = 100+ scoreRetryBias = 1000+)++// A candidate whose probability underflows float32 is omitted by llama-server.+var scoreFloorLogit = float32(math.Log(math.SmallestNonzeroFloat32))++type scoreRowTokens struct {+ tokens, candidates []int+}++func (s *llamaServerRunner) Score(ctx context.Context, input ScoreRequest) (ScoreResponse, error) {+ var result ScoreResponse+ badRequest := func(format string, args ...any) (ScoreResponse, error) {+ return ScoreResponse{}, api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: fmt.Sprintf(format, args...)}+ }+ if len(input.Rows) < 1 || len(input.Rows) > 64 {+ return badRequest("scoring requires 1–64 prompts")+ }+ if input.MaxTokens < 1 || input.MaxTokens > s.ContextLength() {+ return badRequest("max_tokens must be between 1 and %d", s.ContextLength())+ }+ if err := s.sem.Acquire(ctx, 1); err != nil {+ return result, err+ }+ defer s.sem.Release(1)+ status, err := s.getServerStatusRetry(ctx)+ if err != nil {+ return result, err+ } else if status != ServerStatusReady {+ return result, fmt.Errorf("unexpected server status: %s", status)+ }++ rows := make([]scoreRowTokens, len(input.Rows))+ parseSpecial, ordinary := true, false+ for i, row := range input.Rows {+ if len(row.Candidates) < 1 || len(row.Candidates) > 26 {+ return badRequest("prompt %d requires 1–26 candidates", i)+ }+ tokens, err := s.tokenize(ctx, row.Prompt, false, &parseSpecial)+ if err != nil {+ return result, err+ }+ if len(tokens) == 0 || len(tokens) > input.MaxTokens {+ return badRequest("prompt %d has %d tokens; expected 1–%d (input is never truncated)", i, len(tokens), input.MaxTokens)+ }+ // llama-server marks the result truncated when prompt+1 reaches the+ // context limit, so scoring needs two positions of headroom.+ if len(tokens)+2 > s.ContextLength() {+ return badRequest("prompt %d requires %d context tokens for scoring; model has %d", i, len(tokens)+2, s.ContextLength())+ }+ rows[i].tokens = tokens+ result.InputTokens += len(tokens)+ for _, candidate := range row.Candidates {+ joined, err := s.tokenize(ctx, row.Prompt+candidate, false, &parseSpecial)+ if err != nil {+ return result, err+ }+ literal, err := s.tokenize(ctx, candidate, false, &ordinary)+ if err != nil {+ return result, err+ }+ if len(joined) != len(tokens)+1 || !slices.Equal(joined[:len(tokens)], tokens) || len(literal) != 1 || literal[0] != joined[len(tokens)] {+ return badRequest("candidate %q must append exactly one ordinary token to prompt %d", candidate, i)+ }+ id := joined[len(tokens)]+ if id < 0 || slices.Contains(rows[i].candidates, id) {+ return badRequest("prompt %d has invalid or duplicate candidate tokens", i)+ }+ rows[i].candidates = append(rows[i].candidates, id)+ }+ }++ result.OutputTokens, err = s.primeSharedPrefix(ctx, rows)+ if err != nil {+ return ScoreResponse{}, err+ }++ // Rows are scored in order: the first row continues from the primer.+ result.Logits = make([][]float32, len(rows))+ for i, row := range rows {+ logits, outputTokens, err := s.scoreRow(ctx, row)+ if err != nil {+ return ScoreResponse{}, err+ }+ result.Logits[i] = logits+ result.OutputTokens += outputTokens+ }+ return result, nil+}++// llama-server saves a checkpoint of state it cannot roll back, from recurrent+// and sliding-window layers, this many tokens before the end of each prompt it+// evaluates. See https://github.com/ggml-org/llama.cpp/pull/20288.+const scoreCheckpointOffset = 4++// primeSharedPrefix makes llama-server save a checkpoint at the end of the+// tokens every row shares, so each row evaluates only its own last few tokens+// instead of its whole prompt.+//+// Rows share the rendered context and schema and differ only near the end.+// llama-server usually reuses a cached prefix by trimming its cache, but it+// cannot trim the state of recurrent layers, as in hybrid models like Qwen 3.5,+// or of sliding-window layers. Such models can only resume from a checkpoint,+// and the checkpoints llama-server saves on its own sit near the end of each+// prompt, past the point where the next row differs. Without a primer, every row would be+// evaluated from the start.+//+// The primer evaluates the shared tokens plus the first row's next+// scoreCheckpointOffset tokens into the cache, so llama-server saves its+// checkpoint exactly at the end of the shared tokens. The pinned server still+// generates one token with n_predict 0; it is discarded but counted in usage.+// The first row continues from the primer; later rows restore the checkpoint.+// With 8 questions, scoring is about 3x faster.+//+// Priming only changes speed: if llama-server stops saving that checkpoint,+// rows are evaluated from the start and scores stay the same. It can be+// removed once llama-server can save a checkpoint at a requested position or+// score rows as one batch.+func (s *llamaServerRunner) primeSharedPrefix(ctx context.Context, rows []scoreRowTokens) (int, error) {+ if len(rows) < 2 {+ return 0, nil+ }+ first := rows[0].tokens+ shared := len(first)+ for _, row := range rows[1:] {+ shared = min(shared, len(row.tokens))+ for i := range shared {+ if row.tokens[i] != first[i] {+ shared = i+ break+ }+ }+ }+ // Nothing is shared, or the first row is too short to continue from the+ // primer.+ if shared == 0 || len(first) <= shared+scoreCheckpointOffset {+ return 0, nil+ }+ tokens := first[:shared+scoreCheckpointOffset]+ output, err := s.scoreCompletion(ctx, scoreCompletionRequest{Prompt: tokens, CachePrompt: true, NPredict: 0})+ if err != nil {+ return 0, err+ }+ if output.Truncated || output.TokensEvaluated != len(tokens) {+ return 0, fmt.Errorf("scoring did not evaluate the complete shared prefix")+ }+ return output.TokensPredicted, nil+}++// scoreCompletionRequest is the part of llama-server's /completion request+// that scoring uses.+type scoreCompletionRequest struct {+ Prompt []int `json:"prompt"`+ Stream bool `json:"stream"`+ CachePrompt bool `json:"cache_prompt"`+ NPredict int `json:"n_predict"`+ NProbs int `json:"n_probs,omitempty"`+ PostSamplingProbs bool `json:"post_sampling_probs"`+ Samplers []string `json:"samplers,omitempty"`+ TopK int `json:"top_k"`+ Temperature float32 `json:"temperature"`+ LogitBias [][2]float64 `json:"logit_bias,omitempty"`+}++// scoreCompletionResponse is the part of llama-server's /completion response+// that scoring reads.+type scoreCompletionResponse struct {+ TokensPredicted int `json:"tokens_predicted"`+ TokensEvaluated int `json:"tokens_evaluated"`+ Truncated bool `json:"truncated"`+ Probabilities []struct {+ TopProbs []struct {+ ID *int `json:"id"`+ Prob *float64 `json:"prob"`+ } `json:"top_probs"`+ } `json:"completion_probabilities"`+}++// scoreRow reads every candidate from one next-token distribution. Biasing the+// candidates by the same amount keeps their relative logits, so the returned+// probabilities are the softmax over the candidates' original logits.+func (s *llamaServerRunner) scoreRow(ctx context.Context, row scoreRowTokens) ([]float32, int, error) {+ var outputTokens int+ for _, bias := range []float64{scoreBias, scoreRetryBias} {+ req := scoreCompletionRequest{+ Prompt: row.tokens, CachePrompt: true, NPredict: 1,+ NProbs: len(row.candidates), PostSamplingProbs: true,+ Samplers: []string{"top_k", "temperature"}, TopK: len(row.candidates), Temperature: 1,+ }+ for _, id := range row.candidates {+ req.LogitBias = append(req.LogitBias, [2]float64{float64(id), bias})+ }+ output, err := s.scoreCompletion(ctx, req)+ if err != nil {+ return nil, 0, err+ }+ if output.Truncated || output.TokensEvaluated != len(row.tokens) || output.TokensPredicted != 1 || len(output.Probabilities) != 1 || len(output.Probabilities[0].TopProbs) == 0 {+ return nil, 0, fmt.Errorf("scoring did not evaluate the complete prompt and return one distribution")+ }+ outputTokens += output.TokensPredicted+ logits := slices.Repeat([]float32{scoreFloorLogit}, len(row.candidates))+ seen := make([]bool, len(row.candidates))+ outranked := false+ for _, p := range output.Probabilities[0].TopProbs {+ if p.ID == nil || p.Prob == nil || math.IsNaN(*p.Prob) || *p.Prob <= 0 || *p.Prob > 1 {+ return nil, 0, fmt.Errorf("scoring returned an invalid candidate probability")+ }+ j := slices.Index(row.candidates, *p.ID)+ if j < 0 {+ outranked = true+ break+ }+ if seen[j] {+ return nil, 0, fmt.Errorf("scoring returned a candidate twice")+ }+ seen[j] = true+ logits[j] = max(float32(math.Log(*p.Prob)), scoreFloorLogit)+ }+ if !outranked {+ return logits, outputTokens, nil+ }+ }+ return nil, 0, fmt.Errorf("scoring candidates were outranked by other tokens")+}++func (s *llamaServerRunner) scoreCompletion(ctx context.Context, input scoreCompletionRequest) (scoreCompletionResponse, error) {+ var output scoreCompletionResponse+ data, err := json.Marshal(input)+ if err != nil {+ return output, err+ }+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/completion", s.port), bytes.NewReader(data))+ if err != nil {+ return output, err+ }+ req.Header.Set("Content-Type", "application/json")+ res, err := s.httpClient().Do(req)+ if err != nil {+ if ctx.Err() != nil {+ return output, ctx.Err()+ }+ if msg := s.lastErrMsg(); msg != "" {+ return output, fmt.Errorf("scoring failed: %s: %w", msg, err)+ }+ return output, err+ }+ defer res.Body.Close()+ body, err := io.ReadAll(res.Body)+ if err != nil {+ return output, err+ }+ if res.StatusCode != http.StatusOK {+ return output, api.StatusError{StatusCode: res.StatusCode, ErrorMessage: s.statusErrorMessage(body)}+ }+ if err := json.Unmarshal(body, &output); err != nil {+ return output, fmt.Errorf("invalid scoring response: %w", err)+ }+ return output, nil+}diff --git a/llm/llama_server_score_test.go b/llm/llama_server_score_test.gonew file mode 100644index 000000000..931eca103--- /dev/null+++ b/llm/llama_server_score_test.go@@ -0,0 +1,421 @@+package llm++import (+ "cmp"+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "maps"+ "math"+ "net"+ "net/http"+ "net/http/httptest"+ "reflect"+ "slices"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/ollama/ollama/api"+ "golang.org/x/sync/semaphore"+)++func newLlamaScoreTestRunner(t *testing.T, completion http.HandlerFunc) *llamaServerRunner {+ t.Helper()+ mux := http.NewServeMux()+ mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {+ fmt.Fprint(w, `{"status":"ok"}`)+ })+ mux.HandleFunc("/tokenize", func(w http.ResponseWriter, r *http.Request) {+ var req struct {+ Content string `json:"content"`+ AddSpecial bool `json:"add_special"`+ ParseSpecial *bool `json:"parse_special"`+ }+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {+ t.Error(err)+ return+ }+ if req.AddSpecial || req.ParseSpecial == nil {+ t.Error("scoring must explicitly parse special tokens without adding BOS/EOS")+ return+ }+ var tokens []int+ for _, c := range req.Content {+ tokens = append(tokens, int(c))+ }+ if *req.ParseSpecial && strings.HasSuffix(req.Content, "<special>") {+ tokens = append(tokens[:len(tokens)-len("<special>")], 1000)+ }+ // Model a tokenizer that merges the candidate with the prompt's last token.+ if strings.HasSuffix(req.Content, "~") && len(tokens) > 1 {+ tokens = append(tokens[:len(tokens)-2], 1001)+ }+ _ = json.NewEncoder(w).Encode(map[string]any{"tokens": tokens})+ })+ mux.HandleFunc("/completion", completion)+ srv := httptest.NewServer(mux)+ t.Cleanup(srv.Close)+ return &llamaServerRunner{+ port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client(),+ cmd: fakeRunningCmd(), options: api.Options{Runner: api.Runner{NumCtx: 8}},+ sem: semaphore.NewWeighted(1),+ }+}++type scoreCompletionTestRequest struct {+ Prompt []int `json:"prompt"`+ Grammar string `json:"grammar"`+ Stream bool `json:"stream"`+ CachePrompt bool `json:"cache_prompt"`+ NPredict int `json:"n_predict"`+ NProbs int `json:"n_probs"`+ PostSamplingProbs *bool `json:"post_sampling_probs"`+ Temperature float64 `json:"temperature"`+ Samplers []string `json:"samplers"`+ TopK int `json:"top_k"`+ LogitBias [][2]float64 `json:"logit_bias"`+}++// scoreCompletionTestResponse returns the candidates' probabilities in the+// order llama-server sorts them.+func scoreCompletionTestResponse(prompt int, probs map[int]float64) map[string]any {+ ids := slices.Collect(maps.Keys(probs))+ slices.SortFunc(ids, func(a, b int) int { return cmp.Compare(probs[b], probs[a]) })+ var top []any+ for _, id := range ids {+ top = append(top, map[string]any{"id": id, "token": string(rune(id)), "prob": probs[id]})+ }+ return map[string]any{+ "tokens_predicted": 1, "tokens_evaluated": prompt,+ "completion_probabilities": []any{map[string]any{"id": ids[0], "token": string(rune(ids[0])), "prob": probs[ids[0]], "top_probs": top}},+ }+}++func checkScoreRowRequest(t *testing.T, req scoreCompletionTestRequest, candidates []int, bias float64) {+ t.Helper()+ var biased []int+ for _, b := range req.LogitBias {+ if b[1] != bias {+ t.Errorf("candidates must share bias %g: %v", bias, req.LogitBias)+ }+ biased = append(biased, int(b[0]))+ }+ if req.Stream || !req.CachePrompt || req.NPredict != 1 || req.Grammar != "" || req.NProbs != len(candidates) || req.PostSamplingProbs == nil || !*req.PostSamplingProbs ||+ req.Temperature != 1 || !slices.Equal(req.Samplers, []string{"top_k", "temperature"}) || req.TopK != len(candidates) || !slices.Equal(biased, candidates) {+ t.Errorf("request changes the candidate distribution: %+v", req)+ }+}++func TestLlamaServerScore(t *testing.T) {+ var calls int+ runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {+ var req scoreCompletionTestRequest+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {+ t.Error(err)+ return+ }+ calls+++ switch {+ case slices.Equal(req.Prompt, []int{'a', 'b', 'c'}):+ checkScoreRowRequest(t, req, []int{'Z', 'A'}, scoreBias)+ _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), map[int]float64{'Z': 0.25, 'A': 0.75}))+ case slices.Equal(req.Prompt, []int{'x', 'y'}):+ checkScoreRowRequest(t, req, []int{'B'}, scoreBias)+ _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), map[int]float64{'B': 1}))+ default:+ t.Errorf("rendered prompt changed: %v", req.Prompt)+ }+ })+ // Model generation defaults must not affect scoring.+ runner.options.Temperature = 0+ runner.options.Stop = []string{"A"}+ runner.options.RepeatPenalty = 2+ runner.options.NumCtx = 5 // The longest prompt leaves exactly two positions of headroom.+ result, err := runner.Score(t.Context(), ScoreRequest{MaxTokens: 5, Rows: []ScoreRow{+ {Prompt: "abc", Candidates: []string{"Z", "A"}},+ {Prompt: "xy", Candidates: []string{"B"}},+ }})+ if err != nil {+ t.Fatal(err)+ }+ want := [][]float32{{float32(math.Log(0.25)), float32(math.Log(0.75))}, {0}}+ if !reflect.DeepEqual(result.Logits, want) || result.InputTokens != 5 || result.OutputTokens != 2 || calls != 2 {+ t.Fatalf("incomplete or reordered scores/usage: %+v, calls=%d", result, calls)+ }+}++func TestLlamaServerScoreSharedPrefix(t *testing.T) {+ var prompts [][]int+ runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {+ var req scoreCompletionTestRequest+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {+ t.Error(err)+ return+ }+ prompts = append(prompts, req.Prompt)+ if len(prompts) == 1 {+ // The native server generates one token even with n_predict: 0.+ if req.NPredict != 0 || req.NProbs != 0 || len(req.LogitBias) != 0 || len(req.Samplers) != 0 || !req.CachePrompt {+ t.Errorf("unexpected primer request: %+v", req)+ }+ _ = json.NewEncoder(w).Encode(map[string]any{"tokens_predicted": 1, "tokens_evaluated": len(req.Prompt)})+ return+ }+ candidate := int(req.LogitBias[0][0])+ checkScoreRowRequest(t, req, []int{candidate}, scoreBias)+ _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), map[int]float64{candidate: 1}))+ })+ runner.options.NumCtx = 64+ result, err := runner.Score(t.Context(), ScoreRequest{MaxTokens: 32, Rows: []ScoreRow{+ {Prompt: "shared:first", Candidates: []string{"A"}},+ {Prompt: "shared:two", Candidates: []string{"B"}},+ {Prompt: "shared:x", Candidates: []string{"C"}},+ }})+ if err != nil {+ t.Fatal(err)+ }+ // The primer evaluates the shared prefix plus 4 tokens of the first row,+ // which continues from it; the other rows restore the checkpoint.+ want := [][]int{scoreTestTokens("shared:firs"), scoreTestTokens("shared:first"), scoreTestTokens("shared:two"), scoreTestTokens("shared:x")}+ if !reflect.DeepEqual(prompts, want) {+ t.Fatalf("got prompts %v, want %v", prompts, want)+ }+ if !reflect.DeepEqual(result.Logits, [][]float32{{0}, {0}, {0}}) || result.InputTokens != 30 || result.OutputTokens != 4 {+ t.Fatalf("unexpected result: %+v", result)+ }+}++func TestLlamaServerScorePrimer(t *testing.T) {+ for _, tt := range []struct {+ name string+ rows []string+ primer string+ }{+ {"one row", []string{"abcdefgh"}, ""},+ {"no shared prefix", []string{"abcdefgh", "xbcdefgh"}, ""},+ {"first row too short", []string{"abcdefg", "abcdwxyzuv"}, ""},+ {"identical prompts", []string{"abcdefgh", "abcdefgh"}, ""},+ {"shared prefix", []string{"abcdefghij", "abcxyzuvwt", "abcdefgh"}, "abcdefg"},+ } {+ t.Run(tt.name, func(t *testing.T) {+ var primer []int+ runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {+ var req scoreCompletionTestRequest+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {+ t.Error(err)+ return+ }+ if len(req.LogitBias) == 0 {+ primer = req.Prompt+ _ = json.NewEncoder(w).Encode(map[string]any{"tokens_predicted": 1, "tokens_evaluated": len(req.Prompt)})+ return+ }+ _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), map[int]float64{'A': 1}))+ })+ runner.options.NumCtx = 64+ input := ScoreRequest{MaxTokens: 32}+ for _, row := range tt.rows {+ input.Rows = append(input.Rows, ScoreRow{Prompt: row, Candidates: []string{"A"}})+ }+ if _, err := runner.Score(t.Context(), input); err != nil {+ t.Fatal(err)+ }+ if !slices.Equal(primer, scoreTestTokens(tt.primer)) {+ t.Fatalf("primed %v, want %v", primer, scoreTestTokens(tt.primer))+ }+ })+ }+}++// scoreTestTokens tokenizes s the way the test tokenizer does: one token per rune.+func scoreTestTokens(s string) []int {+ var tokens []int+ for _, c := range s {+ tokens = append(tokens, int(c))+ }+ return tokens+}++func TestLlamaServerScoreOutranked(t *testing.T) {+ for _, recovers := range []bool{true, false} {+ t.Run(fmt.Sprintf("recovers=%v", recovers), func(t *testing.T) {+ var calls int+ runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {+ var req scoreCompletionTestRequest+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {+ t.Error(err)+ return+ }+ calls+++ bias := float64(scoreBias)+ if calls > 1 {+ bias = scoreRetryBias+ }+ checkScoreRowRequest(t, req, []int{'A', 'B'}, bias)+ probs := map[int]float64{'A': 0.6, 'x': 0.4}+ if recovers && calls > 1 {+ probs = map[int]float64{'A': 0.6, 'B': 0.4}+ }+ _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), probs))+ })+ result, err := runner.Score(t.Context(), ScoreRequest{MaxTokens: 5, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A", "B"}}}})+ if calls != 2 {+ t.Fatalf("got %d attempts, want 2", calls)+ }+ if !recovers {+ if err == nil {+ t.Fatal("accepted a distribution without every candidate")+ }+ return+ }+ if err != nil || result.OutputTokens != 2 || !reflect.DeepEqual(result.Logits, [][]float32{{float32(math.Log(0.6)), float32(math.Log(0.4))}}) {+ t.Fatalf("unexpected result: %+v, %v", result, err)+ }+ })+ }+}++func TestLlamaServerScoreUnderflow(t *testing.T) {+ runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {+ // llama-server omits candidates whose probability underflows.+ _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(3, map[int]float64{'B': 1}))+ })+ result, err := runner.Score(t.Context(), ScoreRequest{MaxTokens: 5, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A", "B"}}}})+ if err != nil || !reflect.DeepEqual(result.Logits, [][]float32{{scoreFloorLogit, 0}}) {+ t.Fatalf("unexpected result: %+v, %v", result, err)+ }+}++func TestLlamaServerScoreValidation(t *testing.T) {+ for _, tt := range []struct {+ name string+ input ScoreRequest+ }{+ {"empty rows", ScoreRequest{MaxTokens: 7}},+ {"too many rows", ScoreRequest{MaxTokens: 7, Rows: make([]ScoreRow, 65)}},+ {"zero max tokens", ScoreRequest{Rows: []ScoreRow{{Prompt: "a", Candidates: []string{"A"}}}}},+ {"max exceeds context", ScoreRequest{MaxTokens: 9, Rows: []ScoreRow{{Prompt: "a", Candidates: []string{"A"}}}}},+ {"empty prompt", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Candidates: []string{"A"}}}}},+ {"oversize prompt", ScoreRequest{MaxTokens: 2, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A"}}}}},+ {"no scoring token space", ScoreRequest{MaxTokens: 8, Rows: []ScoreRow{{Prompt: "abcdefgh", Candidates: []string{"A"}}}}},+ {"no native scoring headroom", ScoreRequest{MaxTokens: 8, Rows: []ScoreRow{{Prompt: "abcdefg", Candidates: []string{"A"}}}}},+ {"empty candidates", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc"}}}},+ {"too many candidates", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: make([]string, 27)}}}},+ {"multiple tokens", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"AB"}}}}},+ {"special token", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"<special>"}}}}},+ {"changed prefix", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"~"}}}}},+ {"duplicate token", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A", "A"}}}}},+ {"invalid later row", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A"}}, {Prompt: "abc", Candidates: []string{"AB"}}}}},+ } {+ t.Run(tt.name, func(t *testing.T) {+ runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {+ t.Error("started inference before validating every row")+ })+ _, err := runner.Score(t.Context(), tt.input)+ var status api.StatusError+ if !errors.As(err, &status) || status.StatusCode != http.StatusBadRequest {+ t.Fatalf("expected validation error, got %v", err)+ }+ })+ }+}++func TestLlamaServerScoreResponse(t *testing.T) {+ const valid = `{"tokens_predicted":1,"tokens_evaluated":3,"completion_probabilities":[{"id":65,"token":"A","prob":1,"top_probs":[{"id":65,"token":"A","prob":1}]}]}`+ for _, tt := range []struct {+ name string+ body string+ status int+ }{+ {"missing probability", strings.Replace(valid, `"token":"A","prob":1}]`, `"token":"A"}]`, 1), 200},+ {"missing token id", strings.Replace(valid, `[{"id":65,"token":"A","prob":1}]`, `[{"token":"A","prob":1}]`, 1), 200},+ {"zero probability", strings.Replace(valid, `"token":"A","prob":1}]`, `"token":"A","prob":0}]`, 1), 200},+ {"probability above one", strings.Replace(valid, `"token":"A","prob":1}]`, `"token":"A","prob":1.5}]`, 1), 200},+ {"duplicate candidate", strings.Replace(valid, `[{"id":65,"token":"A","prob":1}]`, `[{"id":65,"token":"A","prob":0.5},{"id":65,"token":"A","prob":0.5}]`, 1), 200},+ {"empty top probabilities", strings.Replace(valid, `[{"id":65,"token":"A","prob":1}]`, `[]`, 1), 200},+ {"incomplete prompt", strings.Replace(valid, `"tokens_evaluated":3`, `"tokens_evaluated":2`, 1), 200},+ {"truncated prompt", strings.Replace(valid, `"tokens_predicted":1`, `"truncated":true,"tokens_predicted":1`, 1), 200},+ {"extra generated token", strings.Replace(valid, `"tokens_predicted":1`, `"tokens_predicted":2`, 1), 200},+ {"empty probabilities", `{"tokens_predicted":1,"tokens_evaluated":3,"completion_probabilities":[]}`, 200},+ {"malformed JSON", `{`, 200},+ {"upstream error", `{"error":{"message":"out of memory"}}`, 500},+ } {+ t.Run(tt.name, func(t *testing.T) {+ var failed atomic.Bool+ runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {+ if failed.CompareAndSwap(false, true) {+ w.WriteHeader(tt.status)+ fmt.Fprint(w, tt.body)+ return+ }+ fmt.Fprint(w, valid)+ })+ input := ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A"}}}}+ if _, err := runner.Score(t.Context(), input); err == nil {+ t.Fatal("accepted invalid scoring response")+ } else if tt.status == 500 {+ var status api.StatusError+ if !errors.As(err, &status) || status.StatusCode != 500 || !IsOutOfMemory(err) {+ t.Fatalf("lost upstream status/OOM diagnostic: %v", err)+ }+ }+ ctx, cancel := context.WithTimeout(t.Context(), time.Second)+ defer cancel()+ if _, err := runner.Score(ctx, input); err != nil {+ t.Fatalf("runner unusable after failed scoring: %v", err)+ }+ })+ }+}++func TestLlamaServerScoreCancellation(t *testing.T) {+ for _, queued := range []bool{true, false} {+ t.Run(fmt.Sprintf("queued=%v", queued), func(t *testing.T) {+ started := make(chan struct{})+ var block atomic.Bool+ block.Store(!queued)+ runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {+ if block.CompareAndSwap(true, false) {+ _, _ = io.Copy(io.Discard, r.Body)+ close(started)+ <-r.Context().Done()+ return+ }+ fmt.Fprint(w, `{"tokens_predicted":1,"tokens_evaluated":3,"completion_probabilities":[{"id":65,"token":"A","prob":1,"top_probs":[{"id":65,"token":"A","prob":1}]}]}`)+ })+ input := ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A"}}}}+ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()+ if queued {+ if err := runner.sem.Acquire(t.Context(), 1); err != nil {+ t.Fatal(err)+ }+ cancel()+ } else {+ go func() {+ select {+ case <-started:+ cancel()+ case <-ctx.Done():+ }+ }()+ }+ _, err := runner.Score(ctx, input)+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("cancellation lost: %v", err)+ }+ if queued {+ runner.sem.Release(1)+ }+ next, done := context.WithTimeout(t.Context(), time.Second)+ defer done()+ if _, err := runner.Score(next, input); err != nil {+ t.Fatalf("runner unusable after cancellation: %v", err)+ }+ })+ }+}diff --git a/llm/score.go b/llm/score.gonew file mode 100644index 000000000..e7d10ee76--- /dev/null+++ b/llm/score.go@@ -0,0 +1,27 @@+package llm++import "context"++// Scorer is an optional runner capability for next-token candidate scoring.+// Prompts are already rendered; no chat template or generation options apply.+type Scorer interface {+ Score(context.Context, ScoreRequest) (ScoreResponse, error)+}++type ScoreRequest struct {+ Rows []ScoreRow `json:"rows"`+ MaxTokens int `json:"max_tokens"`+}++type ScoreRow struct {+ Prompt string `json:"prompt"`+ Candidates []string `json:"candidates"`+}++type ScoreResponse struct {+ // Logits may be log probabilities: a shared offset within a row does not+ // affect the softmax over its candidates.+ Logits [][]float32 `json:"logits"`+ InputTokens int `json:"input_tokens"` // Sum of complete prompt lengths, including shared tokens.+ OutputTokens int `json:"output_tokens,omitempty"` // Tokens generated internally to obtain candidate scores.+}diff --git a/server/routes.go b/server/routes.goindex 6ca00a2e0..d56c98c69 100644--- a/server/routes.go+++ b/server/routes.go@@ -33,6 +33,7 @@ import ("github.com/ollama/ollama/api""github.com/ollama/ollama/auth"+ "github.com/ollama/ollama/decision""github.com/ollama/ollama/discover""github.com/ollama/ollama/envconfig""github.com/ollama/ollama/format"@@ -837,6 +838,93 @@ func (s *Server) GenerateHandler(c *gin.Context) {streamResponse(c, ch)}+// SystemOneHandler compiles typed questions, scores their allowed answers, and+// returns probabilities. Callers must select weights trained for the prompt format.+func (s *Server) SystemOneHandler(c *gin.Context) {+ // TODO(parthsareen): Check token limits before copying state and schema into+ // each question's prompt. This byte cap limits memory use until then.+ c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 64<<10)+ var req decision.Request+ if err := c.ShouldBindJSON(&req); err != nil {+ var sizeErr *http.MaxBytesError+ if errors.As(err, &sizeErr) {+ c.JSON(http.StatusRequestEntityTooLarge, gin.H{"error": "request body must not exceed 64 KiB"})+ return+ }+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})+ return+ }+ compiled, err := decision.Compile(req)+ if err != nil {+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})+ return+ }+ ref, err := parseAndValidateModelRef(req.Model)+ if err != nil {+ writeModelRefParseError(c, err, http.StatusNotFound, fmt.Sprintf("model '%s' not found", req.Model))+ return+ }+ if ref.Source == modelSourceCloud {+ c.JSON(http.StatusBadRequest, gin.H{"error": "System One requires a local Nimble or Tev model"})+ return+ }+ name, err := getExistingName(ref.Name)+ if err != nil {+ c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})+ return+ }+ m, err := GetModel(name.String())+ if err != nil {+ handleScheduleError(c, req.Model, err)+ return+ }+ if !m.isGGUF() || m.Config.Renderer != "qwen3.5" && !(m.Config.Renderer == "" && m.Config.ModelFamily == "qwen35") {+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("model %q is not supported by System One; use a local Nimble or Tev GGUF model", req.Model)})+ return+ }+ r, _, _, err := s.scheduleRunner(c.Request.Context(), m, []model.Capability{model.CapabilityCompletion}, nil, req.KeepAlive, nil)+ if err != nil {+ handleScheduleError(c, req.Model, err)+ return+ }+ scorer, ok := r.(llm.Scorer)+ if !ok {+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("model %q does not support System One scoring; use a local Nimble or Tev model with a scoring-capable runner", req.Model)})+ return+ }+ if err := compiled.Render(func(messages []api.Message) (string, error) {+ if m.System != "" {+ messages = append([]api.Message{{Role: "system", Content: m.System}}, messages...)+ }+ think := &api.ThinkValue{Value: false}+ if m.HasChatTemplate && chatModeForModel(m) == chatExecutionModeNative {+ return r.ApplyChatTemplate(c.Request.Context(), llm.ChatRequest{Messages: messages, Think: think})+ }+ return renderPrompt(m, messages, nil, think)+ }); err != nil {+ c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})+ return+ }+ compiled.Request.MaxTokens = r.ContextLength()+ result, err := scorer.Score(c.Request.Context(), compiled.Request)+ if err != nil {+ s.sched.expireRunnersForRuntimeOOM(m, err)+ status := http.StatusInternalServerError+ var statusErr api.StatusError+ if errors.As(err, &statusErr) {+ status = statusErr.StatusCode+ }+ c.JSON(status, gin.H{"error": err.Error()})+ return+ }+ response, err := compiled.Answer(req.Model, result)+ if err != nil {+ c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})+ return+ }+ c.JSON(http.StatusOK, response)+}+func (s *Server) EmbedHandler(c *gin.Context) {checkpointStart := time.Now()var req api.EmbedRequest@@ -1967,6 +2055,7 @@ func (s *Server) GenerateRoutes() (http.Handler, error) {r.POST("/api/chat", s.withInferenceRequestLogging("/api/chat", s.ChatHandler)...)r.POST("/api/embed", s.EmbedHandler)r.POST("/api/embeddings", s.EmbeddingsHandler)+ r.POST("/v1/systemone", s.SystemOneHandler)// Inference (OpenAI compatibility)// TODO(cloud-stage-a): apply Modelfile overlay deltas for local models with clouddiff --git a/server/systemone_test.go b/server/systemone_test.gonew file mode 100644index 000000000..0bede8e56--- /dev/null+++ b/server/systemone_test.go@@ -0,0 +1,204 @@+package server++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"++ "github.com/gin-gonic/gin"+ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/decision"+ gguftest "github.com/ollama/ollama/internal/testutil/gguf"+ "github.com/ollama/ollama/llm"+ "github.com/ollama/ollama/manifest"+ "github.com/ollama/ollama/ml"+ "github.com/ollama/ollama/types/model"+)++type systemOneTestRunner struct {+ mockRunner+ request llm.ScoreRequest+ err error+ calls int+}++func (r *systemOneTestRunner) Score(ctx context.Context, input llm.ScoreRequest) (llm.ScoreResponse, error) {+ r.calls+++ r.request = input+ return llm.ScoreResponse{Logits: [][]float32{{0, 2}}, InputTokens: 123, OutputTokens: 2}, r.err+}++func TestSystemOneHandler(t *testing.T) {+ gin.SetMode(gin.TestMode)+ t.Setenv("OLLAMA_MODELS", t.TempDir())+ config := model.ConfigV2{ModelFormat: "safetensors", Renderer: "qwen3.5", Capabilities: []string{"completion"}}+ createSafetensorsTestModel(t, "safetensors-decision", config, nil)+ for _, modelConfig := range []struct {+ name, architecture, renderer, system, template string+ contextLength int+ }{+ {"decision-test", "qwen35", "qwen3.5", "Model-specific scoring instructions.", "", 1024},+ {"gguf-decision", "qwen35", "", "Native model scoring instructions.", "", 4096},+ {"go-template", "qwen35", "", "Model-specific scoring instructions.", "custom:{{ range .Messages }}{{ .Role }}:{{ .Content }}\n{{ end }}answer:", 1024},+ {"no-system", "qwen35", "qwen3.5", "", "", 1024},+ {"gguf-wrong-architecture", "llama", "", "", "", 1024},+ {"gguf-wrong-renderer", "qwen35", "gemma4", "", "", 1024},+ } {+ kv := gguftest.KV{"general.architecture": modelConfig.architecture}+ if modelConfig.template == "" {+ kv["tokenizer.chat_template"] = "{{ messages }}"+ }+ _, digest := createBinFile(t, kv, nil)+ configLayer, err := createConfigLayer(model.ConfigV2{+ ModelFormat: "gguf", ModelFamily: modelConfig.architecture,+ Renderer: modelConfig.renderer, Capabilities: []string{"completion"},+ })+ if err != nil {+ t.Fatal(err)+ }+ layers := []manifest.Layer{{MediaType: "application/vnd.ollama.image.model", Digest: digest}}+ for _, layer := range []struct{ content, mediaType string }{+ {modelConfig.system, "application/vnd.ollama.image.system"},+ {fmt.Sprintf(`{"num_ctx":%d}`, modelConfig.contextLength), "application/vnd.ollama.image.params"},+ } {+ l, err := manifest.NewLayer(strings.NewReader(layer.content), layer.mediaType)+ if err != nil {+ t.Fatal(err)+ }+ layers = append(layers, l)+ }+ if modelConfig.template != "" {+ l, err := manifest.NewLayer(strings.NewReader(modelConfig.template), "application/vnd.ollama.image.template")+ if err != nil {+ t.Fatal(err)+ }+ layers = append(layers, l)+ }+ if err := manifest.WriteManifest(model.ParseName(modelConfig.name), *configLayer, layers); err != nil {+ t.Fatal(err)+ }+ }++ const prefix = `{"model":"decision-test","state":"`+ const suffix = `","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`+ stateLimit := (64 << 10) - len(prefix) - len(suffix)+ for _, tt := range []struct {+ name string+ body string+ err error+ status int+ calls int+ expire bool+ }{+ {"success", `{"model":"decision-test","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},+ {"GGUF success without renderer", `{"model":"gguf-decision","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},+ {"Modelfile template", `{"model":"go-template","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},+ {"no system prompt", `{"model":"no-system","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},+ {"template failure", `{"model":"gguf-decision","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, errors.New("invalid model template"), 500, 0, false},+ {"GGUF incompatible architecture", `{"model":"gguf-wrong-architecture","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},+ {"GGUF conflicting renderer", `{"model":"gguf-wrong-renderer","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},+ {"runner validation", `{"model":"decision-test","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, api.StatusError{StatusCode: 400, ErrorMessage: "prompt too long"}, 400, 1, false},+ {"runner failure", `{"model":"decision-test","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, errors.New("runner failed"), 500, 1, false},+ {"runtime OOM", `{"model":"decision-test","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, errors.New("out of memory"), 500, 1, true},+ {"invalid schema", `{"model":"decision-test","state":"x","questions":{}}`, nil, 400, 0, false},+ {"missing model", `{"model":"missing","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 404, 0, false},+ {"cloud", `{"model":"decision:cloud","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},+ {"safetensors unsupported", `{"model":"safetensors-decision","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},+ {"bad JSON", `{`, nil, 400, 0, false},+ {"at body limit", prefix + strings.Repeat("x", stateLimit) + suffix, nil, 200, 1, false},+ {"over body limit", prefix + strings.Repeat("x", stateLimit+1) + suffix, nil, 413, 0, false},+ } {+ t.Run(tt.name, func(t *testing.T) {+ runner := &systemOneTestRunner{err: tt.err}+ runner.TemplateFn = func(_ context.Context, req llm.ChatRequest) (string, error) {+ if len(req.Messages) != 2 || req.Messages[0].Role != "system" || req.Messages[0].Content != "Native model scoring instructions." || req.Messages[1].Role != "user" {+ t.Fatalf("model system prompt was not passed to the native template: %+v", req.Messages)+ }+ if req.Think == nil || req.Think.Bool() {+ t.Fatal("scoring must disable thinking")+ }+ return "native:" + req.Messages[1].Content, tt.err+ }+ ref := &runnerRef{llama: runner, refCount: 1, sessionDuration: time.Hour}+ s := newServerWithMockRunner(t, &runner.mockRunner)+ s.sched.loadFn = func(req *LlmRequest, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {+ runner.contextLength = req.opts.NumCtx+ ref.model = req.model+ ref.modelKey = schedulerModelKey(req.model)+ s.sched.loadedMu.Lock()+ s.sched.loaded[ref.modelKey] = ref+ s.sched.loadedMu.Unlock()+ req.successCh <- ref+ return false+ }+ router := gin.New()+ router.POST("/v1/systemone", s.SystemOneHandler)+ w := httptest.NewRecorder()+ req := httptest.NewRequest(http.MethodPost, "/v1/systemone", strings.NewReader(tt.body))+ req.ContentLength = -1+ req.Header.Set("Content-Type", "application/json")+ router.ServeHTTP(w, req)+ if w.Code != tt.status || runner.calls != tt.calls {+ t.Fatalf("status=%d calls=%d body=%s", w.Code, runner.calls, w.Body)+ }+ if tt.calls == 0 && tt.err == nil && ref.model != nil {+ t.Fatal("loaded a runner for a rejected request")+ }+ if tt.calls > 0 {+ want := time.Hour+ if tt.expire {+ want = 0+ }+ ref.refMu.Lock()+ duration := ref.sessionDuration+ ref.refMu.Unlock()+ if duration != want {+ t.Fatalf("runner keep-alive=%v, want %v", duration, want)+ }+ }+ if tt.status == 200 {+ var response struct {+ Answers map[string]struct {+ Noul float64 `json:"noul"`+ } `json:"answers"`+ Usage decision.Usage `json:"usage"`+ }+ if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {+ t.Fatal(err)+ }+ if response.Answers["refund"].Noul < 0.88 || response.Usage.InputTokens != 123 || response.Usage.OutputTokens != 2 {+ t.Fatalf("incorrect scoring response: %s", w.Body)+ }+ prompt := runner.request.Rows[0].Prompt+ wantContext := 1024+ if ref.model.HasGoTemplate {+ if !strings.HasPrefix(prompt, "custom:system:Model-specific scoring instructions.\nuser:") || !strings.HasSuffix(prompt, "answer:") {+ t.Fatalf("scorer did not receive the Modelfile template output: %q", prompt)+ }+ } else if ref.model.Config.Renderer == "" {+ wantContext = 4096+ if !strings.HasPrefix(prompt, `native:{"context":`) {+ t.Fatalf("scorer did not receive the native template output: %q", prompt)+ }+ } else {+ wantPrefix := "<|im_start|>user\n"+ if ref.model.System != "" {+ wantPrefix = "<|im_start|>system\nModel-specific scoring instructions.<|im_end|>\n<|im_start|>user\n"+ }+ if !strings.HasPrefix(prompt, wantPrefix) || !strings.HasSuffix(prompt, "<think>\n\n</think>\n\n") {+ t.Fatalf("scorer did not receive the model's rendered prompt: %q", prompt)+ }+ }+ if runner.request.MaxTokens != wantContext {+ t.Fatalf("scoring context = %d, want model num_ctx %d", runner.request.MaxTokens, wantContext)+ }+ }+ })+ }+}