Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
cli: add decision model support to `ollama run` (#18962)
Jeffrey Morgan7a5ae12ead68
diff --git a/api/client.go b/api/client.goindex 7decf3d0f..2eb66cb9b 100644--- a/api/client.go+++ b/api/client.go@@ -443,6 +443,15 @@ func (c *Client) Heartbeat(ctx context.Context) error {return nil}+// SystemOne answers typed questions about a state with a decision model.+func (c *Client) SystemOne(ctx context.Context, req *SystemOneRequest) (*SystemOneResponse, error) {+ var resp SystemOneResponse+ if err := c.do(ctx, http.MethodPost, "/v1/systemone", req, &resp); err != nil {+ return nil, err+ }+ return &resp, nil+}+// Embed generates embeddings from a model.func (c *Client) Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error) {var resp EmbedResponsediff --git a/api/types.go b/api/types.goindex e19de536f..66b6e15fa 100644--- a/api/types.go+++ b/api/types.go@@ -601,6 +601,48 @@ type Runner struct {DraftNumPredict int `json:"draft_num_predict,omitempty"`}+// SystemOneRequest is the request passed to [Client.SystemOne]. It asks a+// decision model typed questions about a state.+type SystemOneRequest struct {+ Model string `json:"model"`+ Runner string `json:"runner,omitempty"`+ State json.RawMessage `json:"state"`+ Images []ImageData `json:"images,omitempty"`+ Videos []json.RawMessage `json:"videos,omitempty"`+ Questions *SystemOneQuestions `json:"questions"`+ KeepAlive *Duration `json:"keep_alive,omitempty"`+}++// SystemOneQuestions keeps questions in request order, which is the order of+// the answers.+type SystemOneQuestions = orderedmap.Map[string, SystemOneQuestion]++// SystemOneQuestion is a noul (yes/no), choice, or score question. Criteria is+// an object for noul and choice questions and an ordered array for score.+type SystemOneQuestion struct {+ Type string `json:"type"`+ Instructions json.RawMessage `json:"instructions"`+ Criteria json.RawMessage `json:"criteria,omitempty"`+}++// SystemOneResponse is the response returned by [Client.SystemOne]. Each+// answer's type field says which answer fields it has.+type SystemOneResponse struct {+ Model string `json:"model"`+ Answers *SystemOneAnswers `json:"answers"`+ Usage SystemOneUsage `json:"usage"`+ PromptEvalCachedCount *int `json:"prompt_eval_cached_count,omitempty"`+}++// SystemOneAnswers keeps answers in request order.+type SystemOneAnswers = orderedmap.Map[string, json.RawMessage]++// SystemOneUsage counts the tokens evaluated and generated to answer.+type SystemOneUsage struct {+ InputTokens int `json:"input_tokens"`+ OutputTokens int `json:"output_tokens"`+}+// EmbedRequest is the request passed to [Client.Embed].type EmbedRequest struct {// Model is the model name.diff --git a/cmd/cmd.go b/cmd/cmd.goindex 9cd618613..8660c7f71 100644--- a/cmd/cmd.go+++ b/cmd/cmd.go@@ -972,24 +972,15 @@ func RunHandler(cmd *cobra.Command, args []string) error {return err}- audioCapable := slices.Contains(info.Capabilities, model.CapabilityAudio)- opts.MultiModal = slices.Contains(info.Capabilities, model.CapabilityVision) || audioCapable+ applyShowResponseToRunOptions(&opts, info)- // TODO: remove the projector info and vision info checks below,- // these are left in for backwards compatibility with older servers- // that don't have the capabilities field in the model info- if len(info.ProjectorInfo) != 0 {- opts.MultiModal = true+ if slices.Contains(info.Capabilities, model.CapabilityDecision) {+ return runDecision(cmd, opts, interactive)}- for k := range info.ModelInfo {- if strings.Contains(k, ".vision.") {- opts.MultiModal = true- break- }+ if cmd.Flags().Changed("ask") {+ return fmt.Errorf("%s is not a decision model, so it can't answer --ask questions", name)}- applyShowResponseToRunOptions(&opts, info)-// Check if this is an embedding modelisEmbeddingModel := slices.Contains(info.Capabilities, model.CapabilityEmbedding)@@ -1824,6 +1815,8 @@ type runOptions struct {Think *api.ThinkValueHideThinking boolShowConnect bool+ Decision bool+ Questions *api.SystemOneQuestions}func (r runOptions) Copy() runOptions {@@ -1876,12 +1869,27 @@ func (r runOptions) Copy() runOptions {Think: think,HideThinking: r.HideThinking,ShowConnect: r.ShowConnect,+ Decision: r.Decision,+ Questions: r.Questions,}}func applyShowResponseToRunOptions(opts *runOptions, info *api.ShowResponse) {opts.ParentModel = info.Details.ParentModelopts.LoadedMessages = slices.Clone(info.Messages)+ opts.MultiModal = slices.Contains(info.Capabilities, model.CapabilityVision) ||+ slices.Contains(info.Capabilities, model.CapabilityAudio)++ // Keep media input working with older servers that do not report capabilities.+ if len(info.ProjectorInfo) != 0 {+ opts.MultiModal = true+ }+ for key := range info.ModelInfo {+ if strings.Contains(key, ".vision.") {+ opts.MultiModal = true+ break+ }+ }}type displayResponseState struct {@@ -2363,18 +2371,6 @@ func launchInteractiveModel(cmd *cobra.Command, modelName string) error {return err}- audioCapable := slices.Contains(info.Capabilities, model.CapabilityAudio)- opts.MultiModal = slices.Contains(info.Capabilities, model.CapabilityVision) || audioCapable- if len(info.ProjectorInfo) != 0 {- opts.MultiModal = true- }- for key := range info.ModelInfo {- if strings.Contains(key, ".vision.") {- opts.MultiModal = true- break- }- }-applyShowResponseToRunOptions(&opts, info)if err := loadOrUnloadModel(cmd, &opts); err != nil {@@ -2571,8 +2567,18 @@ func NewCLI() *cobra.Command {showCmd.Flags().BoolP("verbose", "v", false, "Show detailed model information")runCmd := &cobra.Command{- Use: "run MODEL [PROMPT]",- Short: "Run a model",+ Use: "run MODEL [PROMPT]",+ Short: "Run a model",+ Example: ` ollama run gemma4+ ollama run gemma4 "Why is the sky blue?"+ ollama run nimble --ask "Is a refund being requested?" < ticket.txt+ ollama run nimble --ask "Which team should handle this?" \+ --choice billing="Payments and refunds" \+ --choice technical="Software problems" < ticket.txt+ ollama run nimble --ask "How urgent is this?" \+ --level "Routine" \+ --level "Soon" \+ --level "Immediate" < ticket.txt`,Args: cobra.MinimumNArgs(1),PreRunE: checkServerHeartbeat,RunE: RunHandler,@@ -2590,6 +2596,10 @@ func NewCLI() *cobra.Command {runCmd.Flags().Bool("hidethinking", false, "Hide thinking output (if provided)")runCmd.Flags().Bool("truncate", false, "For embedding models: truncate inputs exceeding context length (default: true). Set --truncate=false to error instead")runCmd.Flags().Int("dimensions", 0, "Truncate output embeddings to specified dimension (embedding models only)")+ var asked []askedQuestion+ runCmd.Flags().Var(questionFlag{"ask", &asked}, "ask", "Ask a decision model a question, answered yes or no unless --choice or --level follow")+ runCmd.Flags().Var(questionFlag{"choice", &asked}, "choice", `Add a label for the preceding --ask to choose from, e.g. billing="Payments and refunds"`)+ runCmd.Flags().Var(questionFlag{"level", &asked}, "level", "Add a level to score the preceding --ask on, from lowest to highest")stopCmd := &cobra.Command{Use: "stop MODEL",diff --git a/cmd/cmd_test.go b/cmd/cmd_test.goindex 463ea1f1f..433e7c303 100644--- a/cmd/cmd_test.go+++ b/cmd/cmd_test.go@@ -2497,6 +2497,44 @@ func TestApplyShowResponseToRunOptions(t *testing.T) {}}+func TestApplyShowResponseRefreshesMultimodal(t *testing.T) {+ for _, tt := range []struct {+ name string+ info api.ShowResponse+ }{+ {+ name: "decision vision",+ info: api.ShowResponse{Capabilities: []model.Capability{model.CapabilityDecision, model.CapabilityVision}},+ },+ {+ name: "audio",+ info: api.ShowResponse{Capabilities: []model.Capability{model.CapabilityAudio}},+ },+ {+ name: "legacy projector",+ info: api.ShowResponse{ProjectorInfo: map[string]any{"general.architecture": "clip"}},+ },+ {+ name: "legacy vision",+ info: api.ShowResponse{ModelInfo: map[string]any{"mllama.vision.block_count": 32}},+ },+ } {+ t.Run(tt.name, func(t *testing.T) {+ opts := runOptions{}+ applyShowResponseToRunOptions(&opts, &tt.info)+ if !opts.MultiModal {+ t.Fatal("loading a multimodal model did not enable media input")+ }+ applyShowResponseToRunOptions(&opts, &api.ShowResponse{+ Capabilities: []model.Capability{model.CapabilityDecision},+ })+ if opts.MultiModal {+ t.Fatal("loading a text-only model did not disable media input")+ }+ })+ }+}+func TestRunOptions_Copy(t *testing.T) {// Setup test dataoriginalKeepAlive := &api.Duration{Duration: 5 * time.Minute}diff --git a/cmd/decide.go b/cmd/decide.gonew file mode 100644index 000000000..37ba126d2--- /dev/null+++ b/cmd/decide.go@@ -0,0 +1,227 @@+package cmd++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "os"+ "os/signal"+ "strconv"+ "strings"+ "text/tabwriter"+ "time"++ "github.com/spf13/cobra"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/internal/orderedmap"+ "github.com/ollama/ollama/progress"+)++// runDecision runs a decision model, which answers the --ask questions about+// a state: the text after the model name, or stdin.+func runDecision(cmd *cobra.Command, opts runOptions, interactive bool) error {+ opts.Questions = askedQuestions(cmd)+ if interactive {+ opts.Decision = true+ return generateInteractive(cmd, opts)+ }+ if opts.Prompt == "" || opts.Questions.Len() == 0 {+ return fmt.Errorf("%s answers questions about text, e.g. ollama run %s --ask \"Is a refund being requested?\" < ticket.txt", opts.Model, opts.Model)+ }++ input := api.Message{Role: "user", Content: opts.Prompt}+ if opts.MultiModal {+ var err error+ if input.Content, input.Images, err = extractFileData(input.Content); err != nil {+ return err+ }+ }+ return decide(cmd, opts, input, opts.Questions)+}++// yesNoQuestions asks each text as a yes/no question keyed by the text itself.+func yesNoQuestions(texts ...string) *api.SystemOneQuestions {+ questions := &api.SystemOneQuestions{}+ for _, text := range texts {+ instructions, _ := json.Marshal(text)+ questions.Set(text, api.SystemOneQuestion{Type: "noul", Instructions: instructions})+ }+ return questions+}++// questionFlag is --ask, --choice or --level. The three share the questions+// being asked, so each --choice or --level answers the --ask before it.+type questionFlag struct {+ name string+ asked *[]askedQuestion+}++type askedQuestion struct {+ text string+ choices *orderedmap.Map[string, any]+ levels []string+}++func (f questionFlag) String() string { return "" }+func (f questionFlag) Type() string { return "string" }++func (f questionFlag) Set(value string) error {+ if f.name == "ask" {+ *f.asked = append(*f.asked, askedQuestion{text: value})+ return nil+ }+ if len(*f.asked) == 0 {+ return errors.New("must come after the --ask it answers")+ }+ q := &(*f.asked)[len(*f.asked)-1]+ if f.name == "level" {+ q.levels = append(q.levels, value)+ } else {+ if q.choices == nil {+ q.choices = orderedmap.New[string, any]()+ }+ // A label may have a description: billing="Payments and refunds"+ if label, description, ok := strings.Cut(value, "="); ok {+ q.choices.Set(label, description)+ } else {+ q.choices.Set(value, nil)+ }+ }+ if q.choices != nil && q.levels != nil {+ return errors.New("a question takes --choice or --level, not both")+ }+ return nil+}++// askedQuestions returns the --ask questions keyed by their text: yes/no, or+// answered with a --choice label or on a --level scale.+func askedQuestions(cmd *cobra.Command) *api.SystemOneQuestions {+ questions := &api.SystemOneQuestions{}+ flag := cmd.Flags().Lookup("ask")+ if flag == nil {+ return questions+ }+ for _, q := range *flag.Value.(questionFlag).asked {+ question := api.SystemOneQuestion{Type: "noul"}+ question.Instructions, _ = json.Marshal(q.text)+ switch {+ case q.choices != nil:+ question.Type = "choice"+ question.Criteria, _ = json.Marshal(q.choices)+ case q.levels != nil:+ question.Type = "score"+ question.Criteria, _ = json.Marshal(q.levels)+ }+ questions.Set(q.text, question)+ }+ return questions+}++// decide asks a decision model questions about a state and prints each answer:+// the likelier answer to a yes/no question, the chosen label, or the most+// likely score level, with its probability, and for a score the score itself.+func decide(cmd *cobra.Command, opts runOptions, input api.Message, questions *api.SystemOneQuestions) error {+ client, err := api.ClientFromEnvironment()+ if err != nil {+ return err+ }+ // Piped text and multiline messages end in a newline that isn't part of the state.+ text := strings.TrimRight(input.Content, "\r\n")+ state, _ := json.Marshal(text)+ if s := strings.TrimSpace(text); strings.HasPrefix(s, "{") || strings.HasPrefix(s, "[") {+ if json.Valid([]byte(s)) {+ state = json.RawMessage(s)+ }+ }+ req := &api.SystemOneRequest{+ Model: opts.Model,+ Runner: opts.Runner,+ State: state,+ Images: input.Images,+ Questions: questions,+ KeepAlive: opts.KeepAlive,+ }++ ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt)+ defer stop()++ p := progress.NewProgress(os.Stderr)+ p.Add("", progress.NewSpinner(""))+ start := time.Now()+ resp, err := client.SystemOne(ctx, req)+ p.StopAndClear()+ if errors.Is(err, context.Canceled) {+ return nil+ } else if err != nil {+ return err+ }++ if opts.Format == "json" {+ enc := json.NewEncoder(os.Stdout)+ enc.SetEscapeHTML(false)+ return enc.Encode(resp)+ }++ tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)+ for name := range questions.All() {+ raw, ok := resp.Answers.Get(name)+ if !ok {+ return fmt.Errorf("no answer to %q", name)+ }+ var a struct {+ Type string `json:"type"`+ Noul float64 `json:"noul"`+ Choice string `json:"choice"`+ Score float64 `json:"score"`+ Legend map[string]any `json:"legend"`+ Probabilities map[string]float64 `json:"probabilities"`+ }+ if err := json.Unmarshal(raw, &a); err != nil {+ return fmt.Errorf("answer to %q: %w", name, err)+ }+ var answer, measure string+ switch a.Type {+ case "noul":+ answer, measure = "yes", fmt.Sprintf("%.0f%%", a.Noul*100)+ if a.Noul < 0.5 {+ answer, measure = "no", fmt.Sprintf("%.0f%%", (1-a.Noul)*100)+ }+ case "choice":+ answer, measure = a.Choice, fmt.Sprintf("%.0f%%", a.Probabilities[a.Choice]*100)+ case "score":+ // Not the level nearest the score: an answer split between the+ // ends of the scale scores in the middle.+ level := "0"+ for i := 1; i < len(a.Legend); i++ {+ if k := strconv.Itoa(i); a.Probabilities[k] > a.Probabilities[level] {+ level = k+ }+ }+ if v, ok := a.Legend[level]; ok {+ answer = fmt.Sprint(v)+ }+ measure = fmt.Sprintf("%.0f%%\t%.1f / %d", a.Probabilities[level]*100, a.Score, len(a.Legend)-1)+ default:+ return fmt.Errorf("answer to %q has unknown type %q", name, a.Type)+ }+ if questions.Len() > 1 {+ fmt.Fprintf(tw, "%s\t", name)+ }+ fmt.Fprintf(tw, "%s\t%s\n", answer, measure)+ }+ if err := tw.Flush(); err != nil {+ return err+ }++ if verbose, _ := cmd.Flags().GetBool("verbose"); verbose {+ metrics := api.Metrics{+ TotalDuration: time.Since(start),+ PromptEvalCount: resp.Usage.InputTokens,+ PromptEvalCachedCount: resp.PromptEvalCachedCount,+ }+ metrics.Summary()+ }+ return nil+}diff --git a/cmd/decide_test.go b/cmd/decide_test.gonew file mode 100644index 000000000..938a142e7--- /dev/null+++ b/cmd/decide_test.go@@ -0,0 +1,204 @@+package cmd++import (+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "os"+ "strings"+ "testing"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/types/model"+)++// runDecide runs `ollama run args...` with stdin piped in. nimble is a+// decision model and any other model is a chat model.+func runDecide(t *testing.T, stdin string, handler http.HandlerFunc, args ...string) (string, error) {+ t.Helper()+ mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path == "/api/show" && r.Method == http.MethodPost {+ var req api.ShowRequest+ _ = json.NewDecoder(r.Body).Decode(&req)+ capability := model.CapabilityCompletion+ if req.Model == "nimble" {+ capability = model.CapabilityDecision+ }+ _ = json.NewEncoder(w).Encode(api.ShowResponse{Capabilities: []model.Capability{capability}})+ return+ }+ if r.URL.Path == "/v1/systemone" && r.Method == http.MethodPost {+ handler(w, r)+ return+ }+ http.NotFound(w, r)+ }))+ t.Setenv("OLLAMA_HOST", mockServer.URL)+ t.Cleanup(mockServer.Close)++ cmd, _, err := NewCLI().Find([]string{"run"})+ if err != nil {+ t.Fatal(err)+ }+ cmd.SetContext(t.Context())+ if err := cmd.ParseFlags(args); err != nil {+ return "", err+ }++ oldStdin := os.Stdin+ stdinR, stdinW, _ := os.Pipe()+ defer stdinR.Close()+ _, _ = io.WriteString(stdinW, stdin)+ stdinW.Close()+ os.Stdin = stdinR+ defer func() { os.Stdin = oldStdin }()+ out := captureStdout(t, func() {+ err = RunHandler(cmd, cmd.Flags().Args())+ })+ return out, err+}++func TestRunDecisionModel(t *testing.T) {+ const refund = `{"Is a refund being requested?":{"type":"noul","instructions":"Is a refund being requested?"}}`+ for _, tt := range []struct {+ name, stdin string+ args []string+ state string+ questions string+ answers string+ want string+ }{+ {+ name: "yes/no",+ stdin: "I was charged twice\n",+ args: []string{"--ask", "Is a refund being requested?"},+ questions: refund,+ answers: `{"Is a refund being requested?":{"type":"noul","noul":0.97}}`,+ want: "yes 97%\n",+ },+ {+ name: "state as an argument",+ args: []string{"I was charged twice", "--ask", "Is a refund being requested?"},+ questions: refund,+ answers: `{"Is a refund being requested?":{"type":"noul","noul":0.2}}`,+ want: "no 80%\n",+ },+ {+ name: "JSON state",+ args: []string{"--ask", "Is a refund being requested?", `{"ticket": "I was charged twice"}`},+ state: `{"ticket":"I was charged twice"}`,+ questions: refund,+ answers: `{"Is a refund being requested?":{"type":"noul","noul":0.97}}`,+ want: "yes 97%\n",+ },+ {+ name: "labels",+ stdin: "I was charged twice",+ args: []string{"--ask", "Which team should handle this?", "--choice", "billing=Payments and refunds", "--choice", "technical"},+ questions: `{"Which team should handle this?":{"type":"choice","instructions":"Which team should handle this?","criteria":{"billing":"Payments and refunds","technical":null}}}`,+ answers: `{"Which team should handle this?":{"type":"choice","choice":"billing","probabilities":{"billing":0.91,"technical":0.09},"confidence":0.6}}`,+ want: "billing 91%\n",+ },+ {+ name: "levels",+ stdin: "I was charged twice",+ args: []string{"--ask", "How urgent is this?", "--level", "Routine", "--level", "Soon", "--level", "Immediate"},+ questions: `{"How urgent is this?":{"type":"score","instructions":"How urgent is this?","criteria":["Routine","Soon","Immediate"]}}`,+ answers: `{"How urgent is this?":{"type":"score","score":1.83,"legend":{"0":"Routine","1":"Soon","2":"Immediate"},"probabilities":{"0":0.04,"1":0.09,"2":0.87},"confidence":0.7}}`,+ want: "Immediate 87% 1.8 / 2\n",+ },+ {+ name: "most likely level, not the nearest",+ stdin: "I was charged twice",+ args: []string{"--ask", "How urgent is this?", "--level", "Routine", "--level", "Soon", "--level", "Immediate"},+ questions: `{"How urgent is this?":{"type":"score","instructions":"How urgent is this?","criteria":["Routine","Soon","Immediate"]}}`,+ answers: `{"How urgent is this?":{"type":"score","score":1.1,"legend":{"0":"Routine","1":"Soon","2":"Immediate"},"probabilities":{"0":0.45,"1":0,"2":0.55}}}`,+ want: "Immediate 55% 1.1 / 2\n",+ },+ {+ name: "questions in order",+ stdin: "I was charged twice",+ args: []string{+ "--ask", "Is a refund being requested?",+ "--ask", "Which team?", "--choice", "billing", "--choice", "technical",+ "--ask", "How urgent?", "--level", "routine", "--level", "soon", "--level", "immediate",+ },+ questions: `{"Is a refund being requested?":{"type":"noul","instructions":"Is a refund being requested?"},"Which team?":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}},"How urgent?":{"type":"score","instructions":"How urgent?","criteria":["routine","soon","immediate"]}}`,+ answers: `{"How urgent?":{"type":"score","score":1.2,"legend":{"0":"routine","1":"soon","2":"immediate"},"probabilities":{"0":0.1,"1":0.6,"2":0.3}},"Which team?":{"type":"choice","choice":"billing","probabilities":{"billing":0.91,"technical":0.09}},"Is a refund being requested?":{"type":"noul","noul":0.97}}`,+ want: "Is a refund being requested? yes 97%\nWhich team? billing 91%\nHow urgent? soon 60% 1.2 / 2\n",+ },+ } {+ t.Run(tt.name, func(t *testing.T) {+ var got struct {+ Model string `json:"model"`+ Runner string `json:"runner"`+ State json.RawMessage `json:"state"`+ Questions json.RawMessage `json:"questions"`+ }+ out, err := runDecide(t, tt.stdin, func(w http.ResponseWriter, r *http.Request) {+ if err := json.NewDecoder(r.Body).Decode(&got); err != nil {+ t.Error(err)+ }+ _, _ = io.WriteString(w, `{"model":"nimble","answers":`+tt.answers+`}`)+ }, append([]string{"nimble", "--runner", "llamacpp"}, tt.args...)...)+ if err != nil {+ t.Fatal(err)+ }+ if tt.state == "" {+ tt.state = `"I was charged twice"`+ }+ if got.Model != "nimble" || got.Runner != "llamacpp" || string(got.State) != tt.state || string(got.Questions) != tt.questions {+ t.Fatalf("unexpected request: %+v", got)+ }+ if out != tt.want {+ t.Fatalf("got output %q, want %q", out, tt.want)+ }+ })+ }++ const response = `{"model":"nimble","answers":{"Is this urgent?":{"type":"noul","noul":0.9}},"usage":{"input_tokens":12,"output_tokens":1}}`+ out, err := runDecide(t, "", func(w http.ResponseWriter, r *http.Request) {+ _, _ = io.WriteString(w, response)+ }, "nimble", "hello", "--ask", "Is this urgent?", "--format", "json")+ if err != nil || out != response+"\n" {+ t.Fatalf("json output = %q, %v", out, err)+ }+}++func TestRunDecisionModelErrors(t *testing.T) {+ const tooLong = "text and schema must not exceed 64 KiB"+ _, err := runDecide(t, "", func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusRequestEntityTooLarge)+ _, _ = io.WriteString(w, `{"error":"`+tooLong+`"}`)+ }, "nimble", "hello", "--ask", "Is this urgent?")+ var status api.StatusError+ if !errors.As(err, &status) || status.StatusCode != http.StatusRequestEntityTooLarge || status.ErrorMessage != tooLong {+ t.Fatalf("lost server error: %v", err)+ }++ _, err = runDecide(t, "", func(w http.ResponseWriter, r *http.Request) {+ _, _ = io.WriteString(w, `{"model":"nimble","answers":{}}`)+ }, "nimble", "hello", "--ask", "Is this urgent?")+ if err == nil || !strings.Contains(err.Error(), `no answer to "Is this urgent?"`) {+ t.Fatalf("missing answer: %v", err)+ }++ unexpected := func(w http.ResponseWriter, r *http.Request) { t.Error("asked without text and a question") }+ for _, tt := range []struct {+ args []string+ err string+ }{+ {[]string{"nimble"}, "answers questions about text"},+ {[]string{"nimble", "hello"}, "answers questions about text"},+ {[]string{"nimble", "--ask", "Is this urgent?"}, "answers questions about text"},+ {[]string{"nimble", "hello", "--choice", "billing", "--ask", "Which team?"}, "must come after the --ask it answers"},+ {[]string{"nimble", "hello", "--ask", "How urgent?", "--choice", "billing", "--level", "low"}, "--choice or --level, not both"},+ {[]string{"gemma4", "Win a free cruise", "--ask", "Is this spam?"}, "gemma4 is not a decision model"},+ } {+ if _, err := runDecide(t, "", unexpected, tt.args...); err == nil || !strings.Contains(err.Error(), tt.err) {+ t.Errorf("args %q: got %v", tt.args, err)+ }+ }+}diff --git a/cmd/interactive.go b/cmd/interactive.goindex ff7fffbb2..b6892fd25 100644--- a/cmd/interactive.go+++ b/cmd/interactive.go@@ -118,7 +118,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {scanner, err := readline.New(readline.Prompt{Prompt: ">>> ",AltPrompt: "... ",- Placeholder: "Send a message (/? for help)",AltPlaceholder: "Press Enter to send",})if err != nil {@@ -135,8 +134,20 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {var sb strings.Buildervar multiline MultilineStatevar thinkExplicitlySet bool = opts.Think != nil+ // Decision models retain the first message and answer questions about it.+ var decisionState *api.Messagefor {+ switch {+ case !opts.Decision:+ scanner.Prompt.Placeholder = "Send a message (/? for help)"+ case opts.Questions.Len() > 0:+ scanner.Prompt.Placeholder = "Enter text to evaluate (/? for help)"+ case decisionState != nil:+ scanner.Prompt.Placeholder = "Ask yes/no questions about the text, one per line (/? for help)"+ default:+ scanner.Prompt.Placeholder = "Enter text to evaluate, then ask yes/no questions (/? for help)"+ }line, err := scanner.Readline()switch {case errors.Is(err, io.EOF):@@ -234,6 +245,12 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {return err}applyShowResponseToRunOptions(&opts, info)+ opts.Decision = slices.Contains(info.Capabilities, model.CapabilityDecision)+ decisionState = nil+ if opts.Decision {+ // Decision models can't generate; the first answer loads the model.+ continue+ }opts.Think, err = inferThinkingOption(&info.Capabilities, &opts, thinkExplicitlySet)if err != nil {return err@@ -277,6 +294,7 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {fmt.Printf("Created new model '%s'\n", args[1])continuecase strings.HasPrefix(line, "/clear"):+ decisionState = nilopts.Messages = []api.Message{}if opts.System != "" {newMessage := api.Message{Role: "system", Content: opts.System}@@ -513,18 +531,46 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {}if sb.Len() > 0 && multiline == MultilineNone {- newMessage := api.Message{Role: "user", Content: sb.String()}+ input := sb.String()+ sb.Reset()+ if opts.Decision && decisionState != nil {+ // Each line is a question, so pasting several asks them together.+ var questions []string+ for _, line := range strings.Split(input, "\n") {+ if line = strings.TrimSpace(line); line != "" {+ questions = append(questions, line)+ }+ }+ if len(questions) > 0 {+ if err := decide(cmd, opts, *decisionState, yesNoQuestions(questions...)); err != nil {+ fmt.Printf("error: %v\n", err)+ }+ }+ continue+ }++ newMessage := api.Message{Role: "user", Content: input}if opts.MultiModal {- msg, images, err := extractFileData(sb.String())- if err != nil {+ if newMessage.Content, newMessage.Images, err = extractFileData(input); err != nil {return err}-- newMessage.Content = msg- newMessage.Images = images}+ if opts.Decision {+ newMessage.Content = strings.TrimSpace(newMessage.Content)+ switch {+ case newMessage.Content == "" && len(newMessage.Images) == 0:+ case opts.Questions.Len() > 0:+ // With --ask, every message is a state to answer the questions about.+ if err := decide(cmd, opts, newMessage, opts.Questions); err != nil {+ fmt.Printf("error: %v\n", err)+ }+ default:+ decisionState = &newMessage+ }+ continue+ }opts.Messages = append(opts.Messages, newMessage)assistant, err := chat(cmd, opts)@@ -532,7 +578,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {if strings.Contains(err.Error(), "does not support thinking") ||strings.Contains(err.Error(), "invalid think value") {fmt.Printf("error: %v\n", err)- sb.Reset()continue}return err@@ -540,8 +585,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {if assistant != nil {opts.Messages = append(opts.Messages, *assistant)}-- sb.Reset()}}}diff --git a/decision/types.go b/decision/types.goindex 758bddb5e..b2a63ebdc 100644--- a/decision/types.go+++ b/decision/types.go@@ -1,30 +1,18 @@package decisionimport (- "encoding/json"-"github.com/ollama/ollama/api""github.com/ollama/ollama/internal/orderedmap")// Questions preserves request order, which determines the order of scored fields.// Its zero value is ready to use with Set.-type Questions = orderedmap.Map[string, Question]--type Request struct {- Model string `json:"model"`- State json.RawMessage `json:"state"`- Images []api.ImageData `json:"images,omitempty"`- Videos []json.RawMessage `json:"videos,omitempty"`- Questions *Questions `json:"questions"`- KeepAlive *api.Duration `json:"keep_alive,omitempty"`-}+type Questions = api.SystemOneQuestions-type Question struct {- Type string `json:"type"`- Instructions json.RawMessage `json:"instructions"`- Criteria json.RawMessage `json:"criteria"`-}+type (+ Request = api.SystemOneRequest+ Question = api.SystemOneQuestion+)// Choice maps a single-token answer code to a value in the question's schema.type Choice struct {@@ -53,10 +41,7 @@ type Response struct {PromptEvalCachedCount *int `json:"prompt_eval_cached_count,omitempty"`}-type Usage struct {- InputTokens int `json:"input_tokens"`- OutputTokens int `json:"output_tokens"`-}+type Usage = api.SystemOneUsagetype NoulAnswer struct {Type string `json:"type"`diff --git a/docs/capabilities/decision.mdx b/docs/capabilities/decision.mdxindex 8e23702f7..46ea98f47 100644--- a/docs/capabilities/decision.mdx+++ b/docs/capabilities/decision.mdx@@ -23,6 +23,55 @@ Nimble, Tev1, [Clef](https://ollama.com/library/clef), and [Clef Flash](https://Local requests do not need an API key.+## Ask from the terminal++`ollama run` answers questions about a state: text or JSON, piped in or given after the model name. Ask each question with `--ask`. It is answered yes or no, unless you follow it with labels to choose from (`--choice`) or levels to score it on (`--level`).++```shell examples.sh+# Ask a yes/no question+ollama run nimble --ask "Is a refund being requested?" < ticket.txt++# Choose a label+ollama run nimble --ask "Which team should handle this?" \+ --choice billing="Payments and refunds" \+ --choice technical="Software problems" < ticket.txt++# Score on a scale, from the lowest level to the highest+ollama run nimble --ask "How urgent is this?" \+ --level "Routine" \+ --level "Soon" \+ --level "Immediate" < ticket.txt++# Ask several questions at once+ollama run nimble --ask "Is a refund being requested?" \+ --ask "Which team should handle this?" \+ --choice billing="Payments and refunds" \+ --choice technical="Software problems" \+ --ask "How urgent is this?" \+ --level "Routine" \+ --level "Soon" \+ --level "Immediate" < ticket.txt++# Use a JSON file as the state, and refer to its fields in backticks+ollama run nimble --ask 'Is a refund being requested in `body`?' < ticket.json++# Judge an image with Clef+ollama run clef ./receipt.png --ask "Is this a restaurant receipt?"++# Print the full response+ollama run nimble --ask "Is a refund being requested?" --format json < ticket.txt+```++Each answer prints on its own line. A yes/no answer shows the likelier answer and its probability, a label shows its probability, and a score shows the most likely level, its probability, and the score on the scale:++```text+Is a refund being requested? yes 97%+Which team should handle this? billing 91%+How urgent is this? Soon 64% 1.2 / 2+```++Without a state, `ollama run nimble` opens an interactive prompt. Your first message is the state, and after that each line is a yes/no question about it, so you can paste several at once; `/clear` starts over. With `--ask`, each message is a new state to answer those questions about.+## Choose a labelPut your input in `state` and describe the possible answers in `criteria`:diff --git a/docs/cli.mdx b/docs/cli.mdxindex 8ed110eed..ae4215544 100644--- a/docs/cli.mdx+++ b/docs/cli.mdx@@ -79,6 +79,23 @@ Output is a JSON array:echo "Hello world" | ollama run nomic-embed-text```+### Ask a decision model++```+ollama run nimble --ask "Is a refund being requested?" < ticket.txt++ollama run nimble --ask "Which team should handle this?" \+ --choice billing="Payments and refunds" \+ --choice technical="Software problems" < ticket.txt++ollama run nimble --ask "How urgent is this?" \+ --level "Routine" \+ --level "Soon" \+ --level "Immediate" < ticket.txt+```++The state is piped in or given after the model name, as text or JSON. Each `--ask` question is answered yes or no, unless it's followed by labels to choose from (`--choice`) or levels to score it on (`--level`). See [Decision](/capabilities/decision#ask-from-the-terminal).+### Download a model```diff --git a/server/routes.go b/server/routes.goindex e14e036ab..200af3e03 100644--- a/server/routes.go+++ b/server/routes.go@@ -990,7 +990,13 @@ func (s *Server) SystemOneHandler(c *gin.Context) {c.JSON(http.StatusBadRequest, gin.H{"error": "System One requires a local decision model"})return}- m, err := s.getExistingModelForRunner(ref.Name.String(), "")+ if runner, err := normalizeRunner(req.Runner); err != nil {+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})+ return+ } else {+ req.Runner = runner+ }+ m, err := s.getExistingModelForRunner(ref.Name.String(), req.Runner)if err != nil {if errors.Is(err, os.ErrNotExist) {c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})diff --git a/server/systemone_test.go b/server/systemone_test.goindex 50e0c4d89..ce14b7ac6 100644--- a/server/systemone_test.go+++ b/server/systemone_test.go@@ -106,6 +106,59 @@ func TestDecisionModelRejectsCompletion(t *testing.T) {}}+func TestSystemOneHandlerRunnerSelection(t *testing.T) {+ gin.SetMode(gin.TestMode)+ t.Setenv("OLLAMA_MODELS", t.TempDir())+ for _, variant := range []struct{ runner, format string }{+ {manifest.RunnerLlamaCPP, manifest.FormatGGUF},+ {manifest.RunnerMLX, manifest.FormatSafetensors},+ } {+ writeShowManifestVariant(t, "decision-"+variant.runner, variant.runner, variant.format, model.ConfigV2{+ ModelFormat: variant.format, Renderer: "clef", Capabilities: []string{"decision"},+ }, map[string]any{"general.architecture": "clef", "clef.decision.type": "clef"})+ }+ writeManifestListFixture(t, "decision-list",+ manifestListFixtureChild{name: "decision-llamacpp", runner: manifest.RunnerLlamaCPP, format: manifest.FormatGGUF},+ manifestListFixtureChild{name: "decision-mlx", runner: manifest.RunnerMLX, format: manifest.FormatSafetensors},+ )+ defaultModel, err := GetModel("decision-list")+ if err != nil {+ t.Fatal(err)+ }+ for _, tt := range []struct{ runner, want string }{+ {"", defaultModel.Runner},+ {"llamacpp", manifest.RunnerLlamaCPP},+ {"mlx", manifest.RunnerMLX},+ {" llama.cpp ", manifest.RunnerLlamaCPP},+ } {+ t.Run(tt.runner, func(t *testing.T) {+ runner := &systemOneTestRunner{}+ s := newServerWithMockRunner(t, &runner.mockRunner)+ selectedRunner := make(chan string, 1)+ s.sched.loadFn = func(req *LlmRequest, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {+ selectedRunner <- req.model.Runner+ req.successCh <- &runnerRef{llama: runner}+ return false+ }+ w := createRequest(t, s.SystemOneHandler, map[string]any{+ "model": "decision-list", "runner": tt.runner, "state": "refund please",+ "questions": map[string]any{"refund": map[string]any{"type": "noul", "instructions": "Refund requested?"}},+ })+ if w.Code != http.StatusOK {+ t.Fatalf("status=%d body=%s", w.Code, w.Body)+ }+ select {+ case got := <-selectedRunner:+ if got != tt.want {+ t.Fatalf("scheduled runner=%q, want %q", got, tt.want)+ }+ default:+ t.Fatal("no runner was scheduled")+ }+ })+ }+}+func TestSystemOneHandler(t *testing.T) {gin.SetMode(gin.TestMode)t.Setenv("OLLAMA_MODELS", t.TempDir())@@ -190,6 +243,7 @@ func TestSystemOneHandler(t *testing.T) {}tests := []testCase{{"success", `{"model":"decision-test","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},+ {"unknown runner", `{"model":"decision-test","runner":"invalid","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},{"Clef image above text body limit", `{"model":"renamed-clef","state":"x","images":["` + base64.StdEncoding.EncodeToString(make([]byte, 70<<10)) + `"],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 200, 1, false},{"over image body limit", `{"model":"renamed-clef","images":["` + strings.Repeat("A", 32<<20) + `"]}`, nil, 413, 0, false},{"images require a supported encoding", `{"model":"decision-test","state":"x","images":["aW1hZ2U="],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},