Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
testing: test more models with tool calling (#12867)
Patrick Devine76eb7d0fff04
diff --git a/integration/api_test.go b/integration/api_test.goindex 48572085d..5d7acd949 100644--- a/integration/api_test.go+++ b/integration/api_test.go@@ -408,99 +408,3 @@ func TestAPIEmbeddings(t *testing.T) {t.Errorf("zero length embedding response")}}--func TestAPIToolCalling(t *testing.T) {- initialTimeout := 60 * time.Second- streamTimeout := 30 * time.Second- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)- defer cancel()-- client, _, cleanup := InitServerConnection(ctx, t)- defer cleanup()-- modelName := "qwen3:0.6b"- if err := PullIfMissing(ctx, client, modelName); err != nil {- t.Fatalf("pull failed %s", err)- }-- tools := []api.Tool{- {- Type: "function",- Function: api.ToolFunction{- Name: "get_weather",- Description: "Get the current weather in a given location",- Parameters: api.ToolFunctionParameters{- Type: "object",- Required: []string{"location"},- Properties: map[string]api.ToolProperty{- "location": {- Type: api.PropertyType{"string"},- Description: "The city and state, e.g. San Francisco, CA",- },- },- },- },- },- }-- req := api.ChatRequest{- Model: modelName,- Messages: []api.Message{- {- Role: "user",- Content: "Call get_weather with location set to San Francisco.",- },- },- Tools: tools,- Options: map[string]any{- "temperature": 0,- },- }-- stallTimer := time.NewTimer(initialTimeout)- var gotToolCall bool- var lastToolCall api.ToolCall-- fn := func(response api.ChatResponse) error {- if len(response.Message.ToolCalls) > 0 {- gotToolCall = true- lastToolCall = response.Message.ToolCalls[len(response.Message.ToolCalls)-1]- }- if !stallTimer.Reset(streamTimeout) {- return fmt.Errorf("stall was detected while streaming response, aborting")- }- return nil- }-- stream := true- req.Stream = &stream- done := make(chan int)- var genErr error- go func() {- genErr = client.Chat(ctx, &req, fn)- done <- 0- }()-- select {- case <-stallTimer.C:- t.Errorf("tool-calling chat never started. Timed out after: %s", initialTimeout.String())- case <-done:- if genErr != nil {- t.Fatalf("chat failed: %v", genErr)- }-- if !gotToolCall {- t.Fatalf("expected at least one tool call, got none")- }-- if lastToolCall.Function.Name != "get_weather" {- t.Errorf("unexpected tool called: got %q want %q", lastToolCall.Function.Name, "get_weather")- }-- if _, ok := lastToolCall.Function.Arguments["location"]; !ok {- t.Errorf("expected tool arguments to include 'location', got: %s", lastToolCall.Function.Arguments.String())- }- case <-ctx.Done():- t.Error("outer test context done while waiting for tool-calling chat")- }-}diff --git a/integration/tools_test.go b/integration/tools_test.gonew file mode 100644index 000000000..d6b8dfa54--- /dev/null+++ b/integration/tools_test.go@@ -0,0 +1,132 @@+//go:build integration++package integration++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/ollama/ollama/api"+)++func TestAPIToolCalling(t *testing.T) {+ initialTimeout := 60 * time.Second+ streamTimeout := 60 * time.Second+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)+ defer cancel()++ client, _, cleanup := InitServerConnection(ctx, t)+ defer cleanup()++ minVRAM := map[string]uint64{+ "qwen3-vl": 16,+ "gpt-oss:20b": 16,+ "gpt-oss:120b": 70,+ "qwen3": 6,+ "llama3.1": 8,+ "llama3.2": 4,+ "mistral": 6,+ "qwen2.5": 6,+ "qwen2": 6,+ "mistral-nemo": 9,+ "mistral-small": 16,+ "mixtral:8x22b": 80,+ "qwq": 20,+ "granite3.3": 7,+ }++ for _, model := range libraryToolsModels {+ t.Run(model, func(t *testing.T) {+ if v, ok := minVRAM[model]; ok {+ skipUnderMinVRAM(t, v)+ }++ if err := PullIfMissing(ctx, client, model); err != nil {+ t.Fatalf("pull failed %s", err)+ }++ tools := []api.Tool{+ {+ Type: "function",+ Function: api.ToolFunction{+ Name: "get_weather",+ Description: "Get the current weather in a given location",+ Parameters: api.ToolFunctionParameters{+ Type: "object",+ Required: []string{"location"},+ Properties: map[string]api.ToolProperty{+ "location": {+ Type: api.PropertyType{"string"},+ Description: "The city and state, e.g. San Francisco, CA",+ },+ },+ },+ },+ },+ }++ req := api.ChatRequest{+ Model: model,+ Messages: []api.Message{+ {+ Role: "user",+ Content: "Call get_weather with location set to San Francisco.",+ },+ },+ Tools: tools,+ Options: map[string]any{+ "temperature": 0,+ },+ }++ stallTimer := time.NewTimer(initialTimeout)+ var gotToolCall bool+ var lastToolCall api.ToolCall++ fn := func(response api.ChatResponse) error {+ if len(response.Message.ToolCalls) > 0 {+ gotToolCall = true+ lastToolCall = response.Message.ToolCalls[len(response.Message.ToolCalls)-1]+ }+ if !stallTimer.Reset(streamTimeout) {+ return fmt.Errorf("stall was detected while streaming response, aborting")+ }+ return nil+ }++ stream := true+ req.Stream = &stream+ done := make(chan int)+ var genErr error+ go func() {+ genErr = client.Chat(ctx, &req, fn)+ done <- 0+ }()++ select {+ case <-stallTimer.C:+ t.Errorf("tool-calling chat never started. Timed out after: %s", initialTimeout.String())+ case <-done:+ if genErr != nil {+ t.Fatalf("chat failed: %v", genErr)+ }++ if !gotToolCall {+ t.Fatalf("expected at least one tool call, got none")+ }++ if lastToolCall.Function.Name != "get_weather" {+ t.Errorf("unexpected tool called: got %q want %q", lastToolCall.Function.Name, "get_weather")+ }++ if _, ok := lastToolCall.Function.Arguments["location"]; !ok {+ t.Errorf("expected tool arguments to include 'location', got: %s", lastToolCall.Function.Arguments.String())+ }+ case <-ctx.Done():+ t.Error("outer test context done while waiting for tool-calling chat")+ }+ })+ }+}diff --git a/integration/utils_test.go b/integration/utils_test.goindex c0bac5e14..8a362408e 100644--- a/integration/utils_test.go+++ b/integration/utils_test.go@@ -260,6 +260,22 @@ var ("snowflake-arctic-embed","snowflake-arctic-embed2",}+ libraryToolsModels = []string{+ "qwen3-vl",+ "gpt-oss:20b",+ "gpt-oss:120b",+ "qwen3",+ "llama3.1",+ "llama3.2",+ "mistral",+ "qwen2.5",+ "qwen2",+ "mistral-nemo",+ "mistral-small",+ "mixtral:8x22b",+ "qwq",+ "granite3.3",+ }blueSkyPrompt = "why is the sky blue? Be brief but factual in your reply"blueSkyExpected = []string{"rayleigh", "scatter", "atmosphere", "nitrogen", "oxygen", "wavelength", "interact"}