Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

cli: add decision model support to `ollama run` (#18962)

Jeffrey Morgan7a5ae12ead68
diff --git a/api/client.go b/api/client.go
index 7decf3d0f..2eb66cb9b 100644
--- a/api/client.go
+++ b/api/client.go
@@ -443,6 +443,15 @@ func (c *Client) Heartbeat(ctx context.Context) error {
return nil
}
+// SystemOne answers typed questions about a state with a decision model.
+func (c *Client) SystemOne(ctx context.Context, req *SystemOneRequest) (*SystemOneResponse, error) {
+ var resp SystemOneResponse
+ if err := c.do(ctx, http.MethodPost, "/v1/systemone", req, &resp); err != nil {
+ return nil, err
+ }
+ return &resp, nil
+}
+
// Embed generates embeddings from a model.
func (c *Client) Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error) {
var resp EmbedResponse
diff --git a/api/types.go b/api/types.go
index e19de536f..66b6e15fa 100644
--- a/api/types.go
+++ b/api/types.go
@@ -601,6 +601,48 @@ type Runner struct {
DraftNumPredict int `json:"draft_num_predict,omitempty"`
}
+// SystemOneRequest is the request passed to [Client.SystemOne]. It asks a
+// decision model typed questions about a state.
+type SystemOneRequest struct {
+ Model string `json:"model"`
+ Runner string `json:"runner,omitempty"`
+ State json.RawMessage `json:"state"`
+ Images []ImageData `json:"images,omitempty"`
+ Videos []json.RawMessage `json:"videos,omitempty"`
+ Questions *SystemOneQuestions `json:"questions"`
+ KeepAlive *Duration `json:"keep_alive,omitempty"`
+}
+
+// SystemOneQuestions keeps questions in request order, which is the order of
+// the answers.
+type SystemOneQuestions = orderedmap.Map[string, SystemOneQuestion]
+
+// SystemOneQuestion is a noul (yes/no), choice, or score question. Criteria is
+// an object for noul and choice questions and an ordered array for score.
+type SystemOneQuestion struct {
+ Type string `json:"type"`
+ Instructions json.RawMessage `json:"instructions"`
+ Criteria json.RawMessage `json:"criteria,omitempty"`
+}
+
+// SystemOneResponse is the response returned by [Client.SystemOne]. Each
+// answer's type field says which answer fields it has.
+type SystemOneResponse struct {
+ Model string `json:"model"`
+ Answers *SystemOneAnswers `json:"answers"`
+ Usage SystemOneUsage `json:"usage"`
+ PromptEvalCachedCount *int `json:"prompt_eval_cached_count,omitempty"`
+}
+
+// SystemOneAnswers keeps answers in request order.
+type SystemOneAnswers = orderedmap.Map[string, json.RawMessage]
+
+// SystemOneUsage counts the tokens evaluated and generated to answer.
+type SystemOneUsage struct {
+ InputTokens int `json:"input_tokens"`
+ OutputTokens int `json:"output_tokens"`
+}
+
// EmbedRequest is the request passed to [Client.Embed].
type EmbedRequest struct {
// Model is the model name.
diff --git a/cmd/cmd.go b/cmd/cmd.go
index 9cd618613..8660c7f71 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -972,24 +972,15 @@ func RunHandler(cmd *cobra.Command, args []string) error {
return err
}
- audioCapable := slices.Contains(info.Capabilities, model.CapabilityAudio)
- opts.MultiModal = slices.Contains(info.Capabilities, model.CapabilityVision) || audioCapable
+ applyShowResponseToRunOptions(&opts, info)
- // TODO: remove the projector info and vision info checks below,
- // these are left in for backwards compatibility with older servers
- // that don't have the capabilities field in the model info
- if len(info.ProjectorInfo) != 0 {
- opts.MultiModal = true
+ if slices.Contains(info.Capabilities, model.CapabilityDecision) {
+ return runDecision(cmd, opts, interactive)
}
- for k := range info.ModelInfo {
- if strings.Contains(k, ".vision.") {
- opts.MultiModal = true
- break
- }
+ if cmd.Flags().Changed("ask") {
+ return fmt.Errorf("%s is not a decision model, so it can't answer --ask questions", name)
}
- applyShowResponseToRunOptions(&opts, info)
-
// Check if this is an embedding model
isEmbeddingModel := slices.Contains(info.Capabilities, model.CapabilityEmbedding)
@@ -1824,6 +1815,8 @@ type runOptions struct {
Think *api.ThinkValue
HideThinking bool
ShowConnect bool
+ Decision bool
+ Questions *api.SystemOneQuestions
}
func (r runOptions) Copy() runOptions {
@@ -1876,12 +1869,27 @@ func (r runOptions) Copy() runOptions {
Think: think,
HideThinking: r.HideThinking,
ShowConnect: r.ShowConnect,
+ Decision: r.Decision,
+ Questions: r.Questions,
}
}
func applyShowResponseToRunOptions(opts *runOptions, info *api.ShowResponse) {
opts.ParentModel = info.Details.ParentModel
opts.LoadedMessages = slices.Clone(info.Messages)
+ opts.MultiModal = slices.Contains(info.Capabilities, model.CapabilityVision) ||
+ slices.Contains(info.Capabilities, model.CapabilityAudio)
+
+ // Keep media input working with older servers that do not report capabilities.
+ if len(info.ProjectorInfo) != 0 {
+ opts.MultiModal = true
+ }
+ for key := range info.ModelInfo {
+ if strings.Contains(key, ".vision.") {
+ opts.MultiModal = true
+ break
+ }
+ }
}
type displayResponseState struct {
@@ -2363,18 +2371,6 @@ func launchInteractiveModel(cmd *cobra.Command, modelName string) error {
return err
}
- audioCapable := slices.Contains(info.Capabilities, model.CapabilityAudio)
- opts.MultiModal = slices.Contains(info.Capabilities, model.CapabilityVision) || audioCapable
- if len(info.ProjectorInfo) != 0 {
- opts.MultiModal = true
- }
- for key := range info.ModelInfo {
- if strings.Contains(key, ".vision.") {
- opts.MultiModal = true
- break
- }
- }
-
applyShowResponseToRunOptions(&opts, info)
if err := loadOrUnloadModel(cmd, &opts); err != nil {
@@ -2571,8 +2567,18 @@ func NewCLI() *cobra.Command {
showCmd.Flags().BoolP("verbose", "v", false, "Show detailed model information")
runCmd := &cobra.Command{
- Use: "run MODEL [PROMPT]",
- Short: "Run a model",
+ Use: "run MODEL [PROMPT]",
+ Short: "Run a model",
+ Example: ` ollama run gemma4
+ ollama run gemma4 "Why is the sky blue?"
+ ollama run nimble --ask "Is a refund being requested?" < ticket.txt
+ ollama run nimble --ask "Which team should handle this?" \
+ --choice billing="Payments and refunds" \
+ --choice technical="Software problems" < ticket.txt
+ ollama run nimble --ask "How urgent is this?" \
+ --level "Routine" \
+ --level "Soon" \
+ --level "Immediate" < ticket.txt`,
Args: cobra.MinimumNArgs(1),
PreRunE: checkServerHeartbeat,
RunE: RunHandler,
@@ -2590,6 +2596,10 @@ func NewCLI() *cobra.Command {
runCmd.Flags().Bool("hidethinking", false, "Hide thinking output (if provided)")
runCmd.Flags().Bool("truncate", false, "For embedding models: truncate inputs exceeding context length (default: true). Set --truncate=false to error instead")
runCmd.Flags().Int("dimensions", 0, "Truncate output embeddings to specified dimension (embedding models only)")
+ var asked []askedQuestion
+ runCmd.Flags().Var(questionFlag{"ask", &asked}, "ask", "Ask a decision model a question, answered yes or no unless --choice or --level follow")
+ runCmd.Flags().Var(questionFlag{"choice", &asked}, "choice", `Add a label for the preceding --ask to choose from, e.g. billing="Payments and refunds"`)
+ runCmd.Flags().Var(questionFlag{"level", &asked}, "level", "Add a level to score the preceding --ask on, from lowest to highest")
stopCmd := &cobra.Command{
Use: "stop MODEL",
diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go
index 463ea1f1f..433e7c303 100644
--- a/cmd/cmd_test.go
+++ b/cmd/cmd_test.go
@@ -2497,6 +2497,44 @@ func TestApplyShowResponseToRunOptions(t *testing.T) {
}
}
+func TestApplyShowResponseRefreshesMultimodal(t *testing.T) {
+ for _, tt := range []struct {
+ name string
+ info api.ShowResponse
+ }{
+ {
+ name: "decision vision",
+ info: api.ShowResponse{Capabilities: []model.Capability{model.CapabilityDecision, model.CapabilityVision}},
+ },
+ {
+ name: "audio",
+ info: api.ShowResponse{Capabilities: []model.Capability{model.CapabilityAudio}},
+ },
+ {
+ name: "legacy projector",
+ info: api.ShowResponse{ProjectorInfo: map[string]any{"general.architecture": "clip"}},
+ },
+ {
+ name: "legacy vision",
+ info: api.ShowResponse{ModelInfo: map[string]any{"mllama.vision.block_count": 32}},
+ },
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ opts := runOptions{}
+ applyShowResponseToRunOptions(&opts, &tt.info)
+ if !opts.MultiModal {
+ t.Fatal("loading a multimodal model did not enable media input")
+ }
+ applyShowResponseToRunOptions(&opts, &api.ShowResponse{
+ Capabilities: []model.Capability{model.CapabilityDecision},
+ })
+ if opts.MultiModal {
+ t.Fatal("loading a text-only model did not disable media input")
+ }
+ })
+ }
+}
+
func TestRunOptions_Copy(t *testing.T) {
// Setup test data
originalKeepAlive := &api.Duration{Duration: 5 * time.Minute}
diff --git a/cmd/decide.go b/cmd/decide.go
new file mode 100644
index 000000000..37ba126d2
--- /dev/null
+++ b/cmd/decide.go
@@ -0,0 +1,227 @@
+package cmd
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "os/signal"
+ "strconv"
+ "strings"
+ "text/tabwriter"
+ "time"
+
+ "github.com/spf13/cobra"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/internal/orderedmap"
+ "github.com/ollama/ollama/progress"
+)
+
+// runDecision runs a decision model, which answers the --ask questions about
+// a state: the text after the model name, or stdin.
+func runDecision(cmd *cobra.Command, opts runOptions, interactive bool) error {
+ opts.Questions = askedQuestions(cmd)
+ if interactive {
+ opts.Decision = true
+ return generateInteractive(cmd, opts)
+ }
+ if opts.Prompt == "" || opts.Questions.Len() == 0 {
+ return fmt.Errorf("%s answers questions about text, e.g. ollama run %s --ask \"Is a refund being requested?\" < ticket.txt", opts.Model, opts.Model)
+ }
+
+ input := api.Message{Role: "user", Content: opts.Prompt}
+ if opts.MultiModal {
+ var err error
+ if input.Content, input.Images, err = extractFileData(input.Content); err != nil {
+ return err
+ }
+ }
+ return decide(cmd, opts, input, opts.Questions)
+}
+
+// yesNoQuestions asks each text as a yes/no question keyed by the text itself.
+func yesNoQuestions(texts ...string) *api.SystemOneQuestions {
+ questions := &api.SystemOneQuestions{}
+ for _, text := range texts {
+ instructions, _ := json.Marshal(text)
+ questions.Set(text, api.SystemOneQuestion{Type: "noul", Instructions: instructions})
+ }
+ return questions
+}
+
+// questionFlag is --ask, --choice or --level. The three share the questions
+// being asked, so each --choice or --level answers the --ask before it.
+type questionFlag struct {
+ name string
+ asked *[]askedQuestion
+}
+
+type askedQuestion struct {
+ text string
+ choices *orderedmap.Map[string, any]
+ levels []string
+}
+
+func (f questionFlag) String() string { return "" }
+func (f questionFlag) Type() string { return "string" }
+
+func (f questionFlag) Set(value string) error {
+ if f.name == "ask" {
+ *f.asked = append(*f.asked, askedQuestion{text: value})
+ return nil
+ }
+ if len(*f.asked) == 0 {
+ return errors.New("must come after the --ask it answers")
+ }
+ q := &(*f.asked)[len(*f.asked)-1]
+ if f.name == "level" {
+ q.levels = append(q.levels, value)
+ } else {
+ if q.choices == nil {
+ q.choices = orderedmap.New[string, any]()
+ }
+ // A label may have a description: billing="Payments and refunds"
+ if label, description, ok := strings.Cut(value, "="); ok {
+ q.choices.Set(label, description)
+ } else {
+ q.choices.Set(value, nil)
+ }
+ }
+ if q.choices != nil && q.levels != nil {
+ return errors.New("a question takes --choice or --level, not both")
+ }
+ return nil
+}
+
+// askedQuestions returns the --ask questions keyed by their text: yes/no, or
+// answered with a --choice label or on a --level scale.
+func askedQuestions(cmd *cobra.Command) *api.SystemOneQuestions {
+ questions := &api.SystemOneQuestions{}
+ flag := cmd.Flags().Lookup("ask")
+ if flag == nil {
+ return questions
+ }
+ for _, q := range *flag.Value.(questionFlag).asked {
+ question := api.SystemOneQuestion{Type: "noul"}
+ question.Instructions, _ = json.Marshal(q.text)
+ switch {
+ case q.choices != nil:
+ question.Type = "choice"
+ question.Criteria, _ = json.Marshal(q.choices)
+ case q.levels != nil:
+ question.Type = "score"
+ question.Criteria, _ = json.Marshal(q.levels)
+ }
+ questions.Set(q.text, question)
+ }
+ return questions
+}
+
+// decide asks a decision model questions about a state and prints each answer:
+// the likelier answer to a yes/no question, the chosen label, or the most
+// likely score level, with its probability, and for a score the score itself.
+func decide(cmd *cobra.Command, opts runOptions, input api.Message, questions *api.SystemOneQuestions) error {
+ client, err := api.ClientFromEnvironment()
+ if err != nil {
+ return err
+ }
+ // Piped text and multiline messages end in a newline that isn't part of the state.
+ text := strings.TrimRight(input.Content, "\r\n")
+ state, _ := json.Marshal(text)
+ if s := strings.TrimSpace(text); strings.HasPrefix(s, "{") || strings.HasPrefix(s, "[") {
+ if json.Valid([]byte(s)) {
+ state = json.RawMessage(s)
+ }
+ }
+ req := &api.SystemOneRequest{
+ Model: opts.Model,
+ Runner: opts.Runner,
+ State: state,
+ Images: input.Images,
+ Questions: questions,
+ KeepAlive: opts.KeepAlive,
+ }
+
+ ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt)
+ defer stop()
+
+ p := progress.NewProgress(os.Stderr)
+ p.Add("", progress.NewSpinner(""))
+ start := time.Now()
+ resp, err := client.SystemOne(ctx, req)
+ p.StopAndClear()
+ if errors.Is(err, context.Canceled) {
+ return nil
+ } else if err != nil {
+ return err
+ }
+
+ if opts.Format == "json" {
+ enc := json.NewEncoder(os.Stdout)
+ enc.SetEscapeHTML(false)
+ return enc.Encode(resp)
+ }
+
+ tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+ for name := range questions.All() {
+ raw, ok := resp.Answers.Get(name)
+ if !ok {
+ return fmt.Errorf("no answer to %q", name)
+ }
+ var a struct {
+ Type string `json:"type"`
+ Noul float64 `json:"noul"`
+ Choice string `json:"choice"`
+ Score float64 `json:"score"`
+ Legend map[string]any `json:"legend"`
+ Probabilities map[string]float64 `json:"probabilities"`
+ }
+ if err := json.Unmarshal(raw, &a); err != nil {
+ return fmt.Errorf("answer to %q: %w", name, err)
+ }
+ var answer, measure string
+ switch a.Type {
+ case "noul":
+ answer, measure = "yes", fmt.Sprintf("%.0f%%", a.Noul*100)
+ if a.Noul < 0.5 {
+ answer, measure = "no", fmt.Sprintf("%.0f%%", (1-a.Noul)*100)
+ }
+ case "choice":
+ answer, measure = a.Choice, fmt.Sprintf("%.0f%%", a.Probabilities[a.Choice]*100)
+ case "score":
+ // Not the level nearest the score: an answer split between the
+ // ends of the scale scores in the middle.
+ level := "0"
+ for i := 1; i < len(a.Legend); i++ {
+ if k := strconv.Itoa(i); a.Probabilities[k] > a.Probabilities[level] {
+ level = k
+ }
+ }
+ if v, ok := a.Legend[level]; ok {
+ answer = fmt.Sprint(v)
+ }
+ measure = fmt.Sprintf("%.0f%%\t%.1f / %d", a.Probabilities[level]*100, a.Score, len(a.Legend)-1)
+ default:
+ return fmt.Errorf("answer to %q has unknown type %q", name, a.Type)
+ }
+ if questions.Len() > 1 {
+ fmt.Fprintf(tw, "%s\t", name)
+ }
+ fmt.Fprintf(tw, "%s\t%s\n", answer, measure)
+ }
+ if err := tw.Flush(); err != nil {
+ return err
+ }
+
+ if verbose, _ := cmd.Flags().GetBool("verbose"); verbose {
+ metrics := api.Metrics{
+ TotalDuration: time.Since(start),
+ PromptEvalCount: resp.Usage.InputTokens,
+ PromptEvalCachedCount: resp.PromptEvalCachedCount,
+ }
+ metrics.Summary()
+ }
+ return nil
+}
diff --git a/cmd/decide_test.go b/cmd/decide_test.go
new file mode 100644
index 000000000..938a142e7
--- /dev/null
+++ b/cmd/decide_test.go
@@ -0,0 +1,204 @@
+package cmd
+
+import (
+ "encoding/json"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "testing"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/types/model"
+)
+
+// runDecide runs `ollama run args...` with stdin piped in. nimble is a
+// decision model and any other model is a chat model.
+func runDecide(t *testing.T, stdin string, handler http.HandlerFunc, args ...string) (string, error) {
+ t.Helper()
+ mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/api/show" && r.Method == http.MethodPost {
+ var req api.ShowRequest
+ _ = json.NewDecoder(r.Body).Decode(&req)
+ capability := model.CapabilityCompletion
+ if req.Model == "nimble" {
+ capability = model.CapabilityDecision
+ }
+ _ = json.NewEncoder(w).Encode(api.ShowResponse{Capabilities: []model.Capability{capability}})
+ return
+ }
+ if r.URL.Path == "/v1/systemone" && r.Method == http.MethodPost {
+ handler(w, r)
+ return
+ }
+ http.NotFound(w, r)
+ }))
+ t.Setenv("OLLAMA_HOST", mockServer.URL)
+ t.Cleanup(mockServer.Close)
+
+ cmd, _, err := NewCLI().Find([]string{"run"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ cmd.SetContext(t.Context())
+ if err := cmd.ParseFlags(args); err != nil {
+ return "", err
+ }
+
+ oldStdin := os.Stdin
+ stdinR, stdinW, _ := os.Pipe()
+ defer stdinR.Close()
+ _, _ = io.WriteString(stdinW, stdin)
+ stdinW.Close()
+ os.Stdin = stdinR
+ defer func() { os.Stdin = oldStdin }()
+ out := captureStdout(t, func() {
+ err = RunHandler(cmd, cmd.Flags().Args())
+ })
+ return out, err
+}
+
+func TestRunDecisionModel(t *testing.T) {
+ const refund = `{"Is a refund being requested?":{"type":"noul","instructions":"Is a refund being requested?"}}`
+ for _, tt := range []struct {
+ name, stdin string
+ args []string
+ state string
+ questions string
+ answers string
+ want string
+ }{
+ {
+ name: "yes/no",
+ stdin: "I was charged twice\n",
+ args: []string{"--ask", "Is a refund being requested?"},
+ questions: refund,
+ answers: `{"Is a refund being requested?":{"type":"noul","noul":0.97}}`,
+ want: "yes 97%\n",
+ },
+ {
+ name: "state as an argument",
+ args: []string{"I was charged twice", "--ask", "Is a refund being requested?"},
+ questions: refund,
+ answers: `{"Is a refund being requested?":{"type":"noul","noul":0.2}}`,
+ want: "no 80%\n",
+ },
+ {
+ name: "JSON state",
+ args: []string{"--ask", "Is a refund being requested?", `{"ticket": "I was charged twice"}`},
+ state: `{"ticket":"I was charged twice"}`,
+ questions: refund,
+ answers: `{"Is a refund being requested?":{"type":"noul","noul":0.97}}`,
+ want: "yes 97%\n",
+ },
+ {
+ name: "labels",
+ stdin: "I was charged twice",
+ args: []string{"--ask", "Which team should handle this?", "--choice", "billing=Payments and refunds", "--choice", "technical"},
+ questions: `{"Which team should handle this?":{"type":"choice","instructions":"Which team should handle this?","criteria":{"billing":"Payments and refunds","technical":null}}}`,
+ answers: `{"Which team should handle this?":{"type":"choice","choice":"billing","probabilities":{"billing":0.91,"technical":0.09},"confidence":0.6}}`,
+ want: "billing 91%\n",
+ },
+ {
+ name: "levels",
+ stdin: "I was charged twice",
+ args: []string{"--ask", "How urgent is this?", "--level", "Routine", "--level", "Soon", "--level", "Immediate"},
+ questions: `{"How urgent is this?":{"type":"score","instructions":"How urgent is this?","criteria":["Routine","Soon","Immediate"]}}`,
+ answers: `{"How urgent is this?":{"type":"score","score":1.83,"legend":{"0":"Routine","1":"Soon","2":"Immediate"},"probabilities":{"0":0.04,"1":0.09,"2":0.87},"confidence":0.7}}`,
+ want: "Immediate 87% 1.8 / 2\n",
+ },
+ {
+ name: "most likely level, not the nearest",
+ stdin: "I was charged twice",
+ args: []string{"--ask", "How urgent is this?", "--level", "Routine", "--level", "Soon", "--level", "Immediate"},
+ questions: `{"How urgent is this?":{"type":"score","instructions":"How urgent is this?","criteria":["Routine","Soon","Immediate"]}}`,
+ answers: `{"How urgent is this?":{"type":"score","score":1.1,"legend":{"0":"Routine","1":"Soon","2":"Immediate"},"probabilities":{"0":0.45,"1":0,"2":0.55}}}`,
+ want: "Immediate 55% 1.1 / 2\n",
+ },
+ {
+ name: "questions in order",
+ stdin: "I was charged twice",
+ args: []string{
+ "--ask", "Is a refund being requested?",
+ "--ask", "Which team?", "--choice", "billing", "--choice", "technical",
+ "--ask", "How urgent?", "--level", "routine", "--level", "soon", "--level", "immediate",
+ },
+ questions: `{"Is a refund being requested?":{"type":"noul","instructions":"Is a refund being requested?"},"Which team?":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}},"How urgent?":{"type":"score","instructions":"How urgent?","criteria":["routine","soon","immediate"]}}`,
+ answers: `{"How urgent?":{"type":"score","score":1.2,"legend":{"0":"routine","1":"soon","2":"immediate"},"probabilities":{"0":0.1,"1":0.6,"2":0.3}},"Which team?":{"type":"choice","choice":"billing","probabilities":{"billing":0.91,"technical":0.09}},"Is a refund being requested?":{"type":"noul","noul":0.97}}`,
+ want: "Is a refund being requested? yes 97%\nWhich team? billing 91%\nHow urgent? soon 60% 1.2 / 2\n",
+ },
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ var got struct {
+ Model string `json:"model"`
+ Runner string `json:"runner"`
+ State json.RawMessage `json:"state"`
+ Questions json.RawMessage `json:"questions"`
+ }
+ out, err := runDecide(t, tt.stdin, func(w http.ResponseWriter, r *http.Request) {
+ if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
+ t.Error(err)
+ }
+ _, _ = io.WriteString(w, `{"model":"nimble","answers":`+tt.answers+`}`)
+ }, append([]string{"nimble", "--runner", "llamacpp"}, tt.args...)...)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if tt.state == "" {
+ tt.state = `"I was charged twice"`
+ }
+ if got.Model != "nimble" || got.Runner != "llamacpp" || string(got.State) != tt.state || string(got.Questions) != tt.questions {
+ t.Fatalf("unexpected request: %+v", got)
+ }
+ if out != tt.want {
+ t.Fatalf("got output %q, want %q", out, tt.want)
+ }
+ })
+ }
+
+ const response = `{"model":"nimble","answers":{"Is this urgent?":{"type":"noul","noul":0.9}},"usage":{"input_tokens":12,"output_tokens":1}}`
+ out, err := runDecide(t, "", func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.WriteString(w, response)
+ }, "nimble", "hello", "--ask", "Is this urgent?", "--format", "json")
+ if err != nil || out != response+"\n" {
+ t.Fatalf("json output = %q, %v", out, err)
+ }
+}
+
+func TestRunDecisionModelErrors(t *testing.T) {
+ const tooLong = "text and schema must not exceed 64 KiB"
+ _, err := runDecide(t, "", func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusRequestEntityTooLarge)
+ _, _ = io.WriteString(w, `{"error":"`+tooLong+`"}`)
+ }, "nimble", "hello", "--ask", "Is this urgent?")
+ var status api.StatusError
+ if !errors.As(err, &status) || status.StatusCode != http.StatusRequestEntityTooLarge || status.ErrorMessage != tooLong {
+ t.Fatalf("lost server error: %v", err)
+ }
+
+ _, err = runDecide(t, "", func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.WriteString(w, `{"model":"nimble","answers":{}}`)
+ }, "nimble", "hello", "--ask", "Is this urgent?")
+ if err == nil || !strings.Contains(err.Error(), `no answer to "Is this urgent?"`) {
+ t.Fatalf("missing answer: %v", err)
+ }
+
+ unexpected := func(w http.ResponseWriter, r *http.Request) { t.Error("asked without text and a question") }
+ for _, tt := range []struct {
+ args []string
+ err string
+ }{
+ {[]string{"nimble"}, "answers questions about text"},
+ {[]string{"nimble", "hello"}, "answers questions about text"},
+ {[]string{"nimble", "--ask", "Is this urgent?"}, "answers questions about text"},
+ {[]string{"nimble", "hello", "--choice", "billing", "--ask", "Which team?"}, "must come after the --ask it answers"},
+ {[]string{"nimble", "hello", "--ask", "How urgent?", "--choice", "billing", "--level", "low"}, "--choice or --level, not both"},
+ {[]string{"gemma4", "Win a free cruise", "--ask", "Is this spam?"}, "gemma4 is not a decision model"},
+ } {
+ if _, err := runDecide(t, "", unexpected, tt.args...); err == nil || !strings.Contains(err.Error(), tt.err) {
+ t.Errorf("args %q: got %v", tt.args, err)
+ }
+ }
+}
diff --git a/cmd/interactive.go b/cmd/interactive.go
index ff7fffbb2..b6892fd25 100644
--- a/cmd/interactive.go
+++ b/cmd/interactive.go
@@ -118,7 +118,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
scanner, err := readline.New(readline.Prompt{
Prompt: ">>> ",
AltPrompt: "... ",
- Placeholder: "Send a message (/? for help)",
AltPlaceholder: "Press Enter to send",
})
if err != nil {
@@ -135,8 +134,20 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
var sb strings.Builder
var multiline MultilineState
var thinkExplicitlySet bool = opts.Think != nil
+ // Decision models retain the first message and answer questions about it.
+ var decisionState *api.Message
for {
+ switch {
+ case !opts.Decision:
+ scanner.Prompt.Placeholder = "Send a message (/? for help)"
+ case opts.Questions.Len() > 0:
+ scanner.Prompt.Placeholder = "Enter text to evaluate (/? for help)"
+ case decisionState != nil:
+ scanner.Prompt.Placeholder = "Ask yes/no questions about the text, one per line (/? for help)"
+ default:
+ scanner.Prompt.Placeholder = "Enter text to evaluate, then ask yes/no questions (/? for help)"
+ }
line, err := scanner.Readline()
switch {
case errors.Is(err, io.EOF):
@@ -234,6 +245,12 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
return err
}
applyShowResponseToRunOptions(&opts, info)
+ opts.Decision = slices.Contains(info.Capabilities, model.CapabilityDecision)
+ decisionState = nil
+ if opts.Decision {
+ // Decision models can't generate; the first answer loads the model.
+ continue
+ }
opts.Think, err = inferThinkingOption(&info.Capabilities, &opts, thinkExplicitlySet)
if err != nil {
return err
@@ -277,6 +294,7 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
fmt.Printf("Created new model '%s'\n", args[1])
continue
case strings.HasPrefix(line, "/clear"):
+ decisionState = nil
opts.Messages = []api.Message{}
if opts.System != "" {
newMessage := api.Message{Role: "system", Content: opts.System}
@@ -513,18 +531,46 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
}
if sb.Len() > 0 && multiline == MultilineNone {
- newMessage := api.Message{Role: "user", Content: sb.String()}
+ input := sb.String()
+ sb.Reset()
+ if opts.Decision && decisionState != nil {
+ // Each line is a question, so pasting several asks them together.
+ var questions []string
+ for _, line := range strings.Split(input, "\n") {
+ if line = strings.TrimSpace(line); line != "" {
+ questions = append(questions, line)
+ }
+ }
+ if len(questions) > 0 {
+ if err := decide(cmd, opts, *decisionState, yesNoQuestions(questions...)); err != nil {
+ fmt.Printf("error: %v\n", err)
+ }
+ }
+ continue
+ }
+
+ newMessage := api.Message{Role: "user", Content: input}
if opts.MultiModal {
- msg, images, err := extractFileData(sb.String())
- if err != nil {
+ if newMessage.Content, newMessage.Images, err = extractFileData(input); err != nil {
return err
}
-
- newMessage.Content = msg
- newMessage.Images = images
}
+ if opts.Decision {
+ newMessage.Content = strings.TrimSpace(newMessage.Content)
+ switch {
+ case newMessage.Content == "" && len(newMessage.Images) == 0:
+ case opts.Questions.Len() > 0:
+ // With --ask, every message is a state to answer the questions about.
+ if err := decide(cmd, opts, newMessage, opts.Questions); err != nil {
+ fmt.Printf("error: %v\n", err)
+ }
+ default:
+ decisionState = &newMessage
+ }
+ continue
+ }
opts.Messages = append(opts.Messages, newMessage)
assistant, err := chat(cmd, opts)
@@ -532,7 +578,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
if strings.Contains(err.Error(), "does not support thinking") ||
strings.Contains(err.Error(), "invalid think value") {
fmt.Printf("error: %v\n", err)
- sb.Reset()
continue
}
return err
@@ -540,8 +585,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
if assistant != nil {
opts.Messages = append(opts.Messages, *assistant)
}
-
- sb.Reset()
}
}
}
diff --git a/decision/types.go b/decision/types.go
index 758bddb5e..b2a63ebdc 100644
--- a/decision/types.go
+++ b/decision/types.go
@@ -1,30 +1,18 @@
package decision
import (
- "encoding/json"
-
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/internal/orderedmap"
)
// Questions preserves request order, which determines the order of scored fields.
// Its zero value is ready to use with Set.
-type Questions = orderedmap.Map[string, Question]
-
-type Request struct {
- Model string `json:"model"`
- State json.RawMessage `json:"state"`
- Images []api.ImageData `json:"images,omitempty"`
- Videos []json.RawMessage `json:"videos,omitempty"`
- Questions *Questions `json:"questions"`
- KeepAlive *api.Duration `json:"keep_alive,omitempty"`
-}
+type Questions = api.SystemOneQuestions
-type Question struct {
- Type string `json:"type"`
- Instructions json.RawMessage `json:"instructions"`
- Criteria json.RawMessage `json:"criteria"`
-}
+type (
+ Request = api.SystemOneRequest
+ Question = api.SystemOneQuestion
+)
// Choice maps a single-token answer code to a value in the question's schema.
type Choice struct {
@@ -53,10 +41,7 @@ type Response struct {
PromptEvalCachedCount *int `json:"prompt_eval_cached_count,omitempty"`
}
-type Usage struct {
- InputTokens int `json:"input_tokens"`
- OutputTokens int `json:"output_tokens"`
-}
+type Usage = api.SystemOneUsage
type NoulAnswer struct {
Type string `json:"type"`
diff --git a/docs/capabilities/decision.mdx b/docs/capabilities/decision.mdx
index 8e23702f7..46ea98f47 100644
--- a/docs/capabilities/decision.mdx
+++ b/docs/capabilities/decision.mdx
@@ -23,6 +23,55 @@ Nimble, Tev1, [Clef](https://ollama.com/library/clef), and [Clef Flash](https://
Local requests do not need an API key.
+## Ask from the terminal
+
+`ollama run` answers questions about a state: text or JSON, piped in or given after the model name. Ask each question with `--ask`. It is answered yes or no, unless you follow it with labels to choose from (`--choice`) or levels to score it on (`--level`).
+
+```shell examples.sh
+# Ask a yes/no question
+ollama run nimble --ask "Is a refund being requested?" < ticket.txt
+
+# Choose a label
+ollama run nimble --ask "Which team should handle this?" \
+ --choice billing="Payments and refunds" \
+ --choice technical="Software problems" < ticket.txt
+
+# Score on a scale, from the lowest level to the highest
+ollama run nimble --ask "How urgent is this?" \
+ --level "Routine" \
+ --level "Soon" \
+ --level "Immediate" < ticket.txt
+
+# Ask several questions at once
+ollama run nimble --ask "Is a refund being requested?" \
+ --ask "Which team should handle this?" \
+ --choice billing="Payments and refunds" \
+ --choice technical="Software problems" \
+ --ask "How urgent is this?" \
+ --level "Routine" \
+ --level "Soon" \
+ --level "Immediate" < ticket.txt
+
+# Use a JSON file as the state, and refer to its fields in backticks
+ollama run nimble --ask 'Is a refund being requested in `body`?' < ticket.json
+
+# Judge an image with Clef
+ollama run clef ./receipt.png --ask "Is this a restaurant receipt?"
+
+# Print the full response
+ollama run nimble --ask "Is a refund being requested?" --format json < ticket.txt
+```
+
+Each answer prints on its own line. A yes/no answer shows the likelier answer and its probability, a label shows its probability, and a score shows the most likely level, its probability, and the score on the scale:
+
+```text
+Is a refund being requested? yes 97%
+Which team should handle this? billing 91%
+How urgent is this? Soon 64% 1.2 / 2
+```
+
+Without a state, `ollama run nimble` opens an interactive prompt. Your first message is the state, and after that each line is a yes/no question about it, so you can paste several at once; `/clear` starts over. With `--ask`, each message is a new state to answer those questions about.
+
## Choose a label
Put your input in `state` and describe the possible answers in `criteria`:
diff --git a/docs/cli.mdx b/docs/cli.mdx
index 8ed110eed..ae4215544 100644
--- a/docs/cli.mdx
+++ b/docs/cli.mdx
@@ -79,6 +79,23 @@ Output is a JSON array:
echo "Hello world" | ollama run nomic-embed-text
```
+### Ask a decision model
+
+```
+ollama run nimble --ask "Is a refund being requested?" < ticket.txt
+
+ollama run nimble --ask "Which team should handle this?" \
+ --choice billing="Payments and refunds" \
+ --choice technical="Software problems" < ticket.txt
+
+ollama run nimble --ask "How urgent is this?" \
+ --level "Routine" \
+ --level "Soon" \
+ --level "Immediate" < ticket.txt
+```
+
+The state is piped in or given after the model name, as text or JSON. Each `--ask` question is answered yes or no, unless it's followed by labels to choose from (`--choice`) or levels to score it on (`--level`). See [Decision](/capabilities/decision#ask-from-the-terminal).
+
### Download a model
```
diff --git a/server/routes.go b/server/routes.go
index e14e036ab..200af3e03 100644
--- a/server/routes.go
+++ b/server/routes.go
@@ -990,7 +990,13 @@ func (s *Server) SystemOneHandler(c *gin.Context) {
c.JSON(http.StatusBadRequest, gin.H{"error": "System One requires a local decision model"})
return
}
- m, err := s.getExistingModelForRunner(ref.Name.String(), "")
+ if runner, err := normalizeRunner(req.Runner); err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ } else {
+ req.Runner = runner
+ }
+ m, err := s.getExistingModelForRunner(ref.Name.String(), req.Runner)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
diff --git a/server/systemone_test.go b/server/systemone_test.go
index 50e0c4d89..ce14b7ac6 100644
--- a/server/systemone_test.go
+++ b/server/systemone_test.go
@@ -106,6 +106,59 @@ func TestDecisionModelRejectsCompletion(t *testing.T) {
}
}
+func TestSystemOneHandlerRunnerSelection(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+ for _, variant := range []struct{ runner, format string }{
+ {manifest.RunnerLlamaCPP, manifest.FormatGGUF},
+ {manifest.RunnerMLX, manifest.FormatSafetensors},
+ } {
+ writeShowManifestVariant(t, "decision-"+variant.runner, variant.runner, variant.format, model.ConfigV2{
+ ModelFormat: variant.format, Renderer: "clef", Capabilities: []string{"decision"},
+ }, map[string]any{"general.architecture": "clef", "clef.decision.type": "clef"})
+ }
+ writeManifestListFixture(t, "decision-list",
+ manifestListFixtureChild{name: "decision-llamacpp", runner: manifest.RunnerLlamaCPP, format: manifest.FormatGGUF},
+ manifestListFixtureChild{name: "decision-mlx", runner: manifest.RunnerMLX, format: manifest.FormatSafetensors},
+ )
+ defaultModel, err := GetModel("decision-list")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, tt := range []struct{ runner, want string }{
+ {"", defaultModel.Runner},
+ {"llamacpp", manifest.RunnerLlamaCPP},
+ {"mlx", manifest.RunnerMLX},
+ {" llama.cpp ", manifest.RunnerLlamaCPP},
+ } {
+ t.Run(tt.runner, func(t *testing.T) {
+ runner := &systemOneTestRunner{}
+ s := newServerWithMockRunner(t, &runner.mockRunner)
+ selectedRunner := make(chan string, 1)
+ s.sched.loadFn = func(req *LlmRequest, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {
+ selectedRunner <- req.model.Runner
+ req.successCh <- &runnerRef{llama: runner}
+ return false
+ }
+ w := createRequest(t, s.SystemOneHandler, map[string]any{
+ "model": "decision-list", "runner": tt.runner, "state": "refund please",
+ "questions": map[string]any{"refund": map[string]any{"type": "noul", "instructions": "Refund requested?"}},
+ })
+ if w.Code != http.StatusOK {
+ t.Fatalf("status=%d body=%s", w.Code, w.Body)
+ }
+ select {
+ case got := <-selectedRunner:
+ if got != tt.want {
+ t.Fatalf("scheduled runner=%q, want %q", got, tt.want)
+ }
+ default:
+ t.Fatal("no runner was scheduled")
+ }
+ })
+ }
+}
+
func TestSystemOneHandler(t *testing.T) {
gin.SetMode(gin.TestMode)
t.Setenv("OLLAMA_MODELS", t.TempDir())
@@ -190,6 +243,7 @@ func TestSystemOneHandler(t *testing.T) {
}
tests := []testCase{
{"success", `{"model":"decision-test","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},
+ {"unknown runner", `{"model":"decision-test","runner":"invalid","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},
{"Clef image above text body limit", `{"model":"renamed-clef","state":"x","images":["` + base64.StdEncoding.EncodeToString(make([]byte, 70<<10)) + `"],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 200, 1, false},
{"over image body limit", `{"model":"renamed-clef","images":["` + strings.Repeat("A", 32<<20) + `"]}`, nil, 413, 0, false},
{"images require a supported encoding", `{"model":"decision-test","state":"x","images":["aW1hZ2U="],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},