Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

test: revamp integration test entrpoints (#16560)

Daniel Hiltgena51df815734e
diff --git a/integration/README.md b/integration/README.md
index 94ac72614..09cd943c7 100644
--- a/integration/README.md
+++ b/integration/README.md
@@ -2,8 +2,21 @@
This directory contains integration tests to exercise Ollama end-to-end to verify behavior
-By default, these tests are disabled so `go test ./...` will exercise only unit tests. To run integration tests you must pass the integration tag. `go test -tags=integration ./...` Some tests require additional tags to enable to allow scoped testing to keep the duration reasonable. For example, testing a broad set of models requires `-tags=integration,models` and a longer timeout (~60m or more depending on the speed of your GPU.). To view the current set of tag combinations use `find integration -type f | xargs grep "go:build"`
+By default, these tests are disabled so `go test ./...` will exercise only unit tests. To run integration tests, pass the `integration` tag and one of the scoped tags:
+```bash
+go test -tags=integration,fast -v -count 1 ./integration/
+go test -tags=integration,release -v -count 1 -timeout 30m ./integration/
+go test -tags=integration,library -v -count 1 -timeout 120m ./integration/
+```
+
+Tags:
+
+- `fast`: quick runner/model smoke coverage.
+- `release`: release regression coverage.
+- `library`: broad library coverage requiring about 2.5 TiB of disk space.
+
+Scope wiring and model selections live in `integration/reg_fast_test.go`, `integration/reg_release_test.go`, and `integration/reg_library_test.go`.
The integration tests have 2 modes of operating.
@@ -21,12 +34,12 @@ harness starts the server.
## Testing a New Model
When implementing new model architecture, use `OLLAMA_TEST_MODEL` to run the
-integration suite against your model.
+integration suite against your model with either the `fast` or `release` coverage.
```bash
# Build the binary first
go build .
# Run integration tests against it
-OLLAMA_TEST_MODEL=mymodel go test -tags integration -v -count 1 -timeout 15m ./integration/
+OLLAMA_TEST_MODEL=mymodel go test -tags=integration,fast -v -count 1 ./integration/
```
diff --git a/integration/api_test.go b/integration/api_test.go
index b6fef6cf8..0a94d0121 100644
--- a/integration/api_test.go
+++ b/integration/api_test.go
@@ -14,6 +14,13 @@ import (
"github.com/ollama/ollama/api"
)
+const (
+ apiTestTimeout = 4 * time.Minute
+ apiInitialResponseTimeout = time.Minute
+ apiOverrideInitialResponseTimeout = 2 * time.Minute
+ apiStreamResponseTimeout = 30 * time.Second
+)
+
func assertBytesMatchToken(t *testing.T, label, token string, ints []int) {
t.Helper()
@@ -31,10 +38,12 @@ func assertBytesMatchToken(t *testing.T, label, token string, ints []int) {
}
}
-func TestAPIGenerate(t *testing.T) {
- initialTimeout := 60 * time.Second
- streamTimeout := 30 * time.Second
- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+func runAPIGenerate(t *testing.T) {
+ initialTimeout := apiInitialResponseTimeout
+ if testModel != "" {
+ initialTimeout = apiOverrideInitialResponseTimeout
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), apiTestTimeout)
defer cancel()
// Set up the test data
req := api.GenerateRequest{
@@ -45,7 +54,6 @@ func TestAPIGenerate(t *testing.T) {
"seed": 123,
},
}
-
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
pullOrSkip(ctx, t, client, req.Model)
@@ -105,7 +113,7 @@ func TestAPIGenerate(t *testing.T) {
} // else incremental response, nothing to check right now...
buf.Write([]byte(response.Response))
- if !stallTimer.Reset(streamTimeout) {
+ if !stallTimer.Reset(apiStreamResponseTimeout) {
return fmt.Errorf("stall was detected while streaming response, aborting")
}
return nil
@@ -188,10 +196,12 @@ func TestAPIGenerate(t *testing.T) {
}
}
-func TestAPIChat(t *testing.T) {
- initialTimeout := 60 * time.Second
- streamTimeout := 30 * time.Second
- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+func runAPIChat(t *testing.T) {
+ initialTimeout := apiInitialResponseTimeout
+ if testModel != "" {
+ initialTimeout = apiOverrideInitialResponseTimeout
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), apiTestTimeout)
defer cancel()
// Set up the test data
req := api.ChatRequest{
@@ -207,7 +217,6 @@ func TestAPIChat(t *testing.T) {
"seed": 123,
},
}
-
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
pullOrSkip(ctx, t, client, req.Model)
@@ -265,7 +274,7 @@ func TestAPIChat(t *testing.T) {
}
} // else incremental response, nothing to check right now...
buf.Write([]byte(response.Message.Content))
- if !stallTimer.Reset(streamTimeout) {
+ if !stallTimer.Reset(apiStreamResponseTimeout) {
return fmt.Errorf("stall was detected while streaming response, aborting")
}
return nil
@@ -310,11 +319,11 @@ func TestAPIChat(t *testing.T) {
}
}
-func TestAPIListModels(t *testing.T) {
+func runAPIListModels(t *testing.T) {
if testModel != "" {
t.Skip("skipping metadata test with model override")
}
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ ctx, cancel := context.WithTimeout(context.Background(), apiTestTimeout)
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
@@ -331,44 +340,54 @@ func TestAPIListModels(t *testing.T) {
if len(resp.Models) == 0 {
t.Fatalf("list should not be empty")
}
- model := resp.Models[0]
+
+ var model *api.ListModelResponse
+ for i := range resp.Models {
+ if resp.Models[i].Name == smol || resp.Models[i].Model == smol || strings.Contains(resp.Models[i].Name, smol) || strings.Contains(resp.Models[i].Model, smol) {
+ model = &resp.Models[i]
+ break
+ }
+ }
+ if model == nil {
+ t.Fatalf("list should include pulled model %s: %#v", smol, resp.Models)
+ }
if model.Name == "" {
- t.Errorf("first model name empty: %#v", model)
+ t.Errorf("model name empty: %#v", model)
}
var nilTime time.Time
if model.ModifiedAt == nilTime {
- t.Errorf("first model modified_at empty: %#v", model)
+ t.Errorf("model modified_at empty: %#v", model)
}
if model.Size == 0 {
- t.Errorf("first model size empty: %#v", model)
+ t.Errorf("model size empty: %#v", model)
}
if model.Digest == "" {
- t.Errorf("first model digest empty: %#v", model)
+ t.Errorf("model digest empty: %#v", model)
}
verifyModelDetails(t, model.Details)
}
func verifyModelDetails(t *testing.T, details api.ModelDetails) {
if details.Format == "" {
- t.Errorf("first model details.format empty: %#v", details)
+ t.Errorf("model details.format empty: %#v", details)
}
if details.Family == "" {
- t.Errorf("first model details.family empty: %#v", details)
+ t.Errorf("model details.family empty: %#v", details)
}
if details.ParameterSize == "" {
- t.Errorf("first model details.parameter_size empty: %#v", details)
+ t.Errorf("model details.parameter_size empty: %#v", details)
}
if details.QuantizationLevel == "" {
- t.Errorf("first model details.quantization_level empty: %#v", details)
+ t.Errorf("model details.quantization_level empty: %#v", details)
}
}
-func TestAPIShowModel(t *testing.T) {
+func runAPIShowModel(t *testing.T) {
if testModel != "" {
t.Skip("skipping metadata test with model override")
}
modelName := "llama3.2"
- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+ ctx, cancel := context.WithTimeout(context.Background(), apiTestTimeout)
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
@@ -405,8 +424,8 @@ func TestAPIShowModel(t *testing.T) {
}
}
-func TestAPIGenerateLogprobs(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+func runAPIGenerateLogprobs(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), apiTestTimeout)
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
@@ -518,8 +537,8 @@ func TestAPIGenerateLogprobs(t *testing.T) {
}
}
-func TestAPIChatLogprobs(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+func runAPIChatLogprobs(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), apiTestTimeout)
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
diff --git a/integration/audio_test.go b/integration/audio_test.go
index 6db67150f..3e9b4f96e 100644
--- a/integration/audio_test.go
+++ b/integration/audio_test.go
@@ -18,12 +18,6 @@ import (
"github.com/ollama/ollama/api"
)
-var defaultAudioModels = []string{
- "nemotron3:33b",
- "gemma4:e2b",
- "gemma4:e4b",
-}
-
// decodeTestAudio returns the test audio clip ("Why is the sky blue?", 16kHz mono WAV).
func decodeTestAudio(t *testing.T) api.ImageData {
t.Helper()
@@ -37,61 +31,56 @@ func decodeTestAudio(t *testing.T) api.ImageData {
// setupAudioModel pulls the model, preloads it, and skips if it doesn't support audio.
func setupAudioModel(ctx context.Context, t *testing.T, client *api.Client, model string) {
t.Helper()
- if testModel == "" {
- pullOrSkip(ctx, t, client, model)
- }
+ pullOrSkip(ctx, t, client, model)
skipIfModelTooLargeForVRAM(ctx, t, client, model)
requireCapability(ctx, t, client, model, "audio")
- err := client.Generate(ctx, &api.GenerateRequest{Model: model}, func(response api.GenerateResponse) error { return nil })
- if err != nil {
- t.Fatalf("failed to load model %s: %s", model, err)
- }
+ preloadGenerateModel(ctx, t, client, api.GenerateRequest{Model: model})
}
-// TestAudioTranscription tests that the model can transcribe audio to text.
-func TestAudioTranscription(t *testing.T) {
- for _, model := range testModels(defaultAudioModels) {
- t.Run(model, func(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
- defer cancel()
- client, _, cleanup := InitServerConnection(ctx, t)
- defer cleanup()
-
- setupAudioModel(ctx, t, client, model)
- audio := decodeTestAudio(t)
- noThink := &api.ThinkValue{Value: false}
-
- req := api.ChatRequest{
- Model: model,
- Think: noThink,
- Messages: []api.Message{
- {
- Role: "system",
- Content: "Transcribe the audio exactly as spoken. Output only the spoken words. Do not answer any question in the audio.",
- },
- {
- Role: "user",
- Content: "What exact words are spoken in this audio?",
- Images: []api.ImageData{audio},
- },
- },
- Stream: &stream,
- Options: map[string]any{
- "temperature": 0,
- "seed": 123,
- "num_predict": 50,
- },
- }
+func registerAudioTranscriptionCases(models []string) {
+ registerModelIntegrationCases("audio-transcription", models, runAudioTranscriptionModel)
+}
- // The audio says "Why is the sky blue?" — expect key words in transcription.
- DoChat(ctx, t, client, req, []string{"sky", "blue"}, 60*time.Second, 10*time.Second)
- })
+func runAudioTranscriptionModel(t *testing.T, model string) {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ defer cancel()
+ client, _, cleanup := InitServerConnection(ctx, t)
+ defer cleanup()
+
+ setupAudioModel(ctx, t, client, model)
+ audio := decodeTestAudio(t)
+ noThink := &api.ThinkValue{Value: false}
+
+ req := api.ChatRequest{
+ Model: model,
+ Think: noThink,
+ Messages: []api.Message{
+ {
+ Role: "system",
+ Content: "Transcribe the audio exactly as spoken. Output only the spoken words. Do not answer any question in the audio.",
+ },
+ {
+ Role: "user",
+ Content: "What exact words are spoken in this audio?",
+ Images: []api.ImageData{audio},
+ },
+ },
+ Stream: &stream,
+ Options: map[string]any{
+ "temperature": 0,
+ "seed": 123,
+ "num_predict": 50,
+ },
}
+
+ // The audio says "Why is the sky blue?" - expect key words in transcription.
+ DoChat(ctx, t, client, req, []string{"sky", "blue"}, 60*time.Second, 10*time.Second)
}
-// TestAudioResponse tests that the model can respond to a spoken question.
-func TestAudioResponse(t *testing.T) {
- for _, model := range testModels(defaultAudioModels) {
+// runAudioResponse tests that the model can respond to a spoken question.
+func runAudioResponse(t *testing.T, models []string) {
+ for _, model := range testModels(models) {
t.Run(model, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
@@ -128,9 +117,9 @@ func TestAudioResponse(t *testing.T) {
}
}
-// TestOpenAIAudioTranscription tests the /v1/audio/transcriptions endpoint.
-func TestOpenAIAudioTranscription(t *testing.T) {
- for _, model := range testModels(defaultAudioModels) {
+// runOpenAIAudioTranscription tests the /v1/audio/transcriptions endpoint.
+func runOpenAIAudioTranscription(t *testing.T, models []string) {
+ for _, model := range testModels(models) {
t.Run(model, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
@@ -182,9 +171,9 @@ func TestOpenAIAudioTranscription(t *testing.T) {
}
}
-// TestOpenAIChatWithAudio tests /v1/chat/completions with input_audio content.
-func TestOpenAIChatWithAudio(t *testing.T) {
- for _, model := range testModels(defaultAudioModels) {
+// runOpenAIChatWithAudio tests /v1/chat/completions with input_audio content.
+func runOpenAIChatWithAudio(t *testing.T, models []string) {
+ for _, model := range testModels(models) {
t.Run(model, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
diff --git a/integration/basic_test.go b/integration/basic_test.go
index b123ad3e3..377f5f568 100644
--- a/integration/basic_test.go
+++ b/integration/basic_test.go
@@ -13,8 +13,8 @@ import (
"github.com/ollama/ollama/api"
)
-func TestBlueSky(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+func runBlueSky(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute)
defer cancel()
// Set up the test data
req := api.ChatRequest{
@@ -34,17 +34,17 @@ func TestBlueSky(t *testing.T) {
ChatTestHelper(ctx, t, req, blueSkyExpected)
}
-func TestUnicode(t *testing.T) {
+func runUnicode(t *testing.T, model string) {
if testModel != "" {
t.Skip("uses hardcoded model, not applicable with model override")
}
- skipUnderMinVRAM(t, 12) // Actual model load is ~26G
+ skipRegisteredMinVRAM(t, model)
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute)
defer cancel()
// Set up the test data
req := api.ChatRequest{
// DeepSeek has a Unicode tokenizer regex, making it a unicode torture test
- Model: "deepseek-coder-v2:16b-lite-instruct-q2_K", // TODO is there an ollama-engine model we can switch to and keep the coverage?
+ Model: model, // TODO is there an ollama-engine model we can switch to and keep the coverage?
Messages: []api.Message{
{
Role: "user",
@@ -63,11 +63,7 @@ func TestUnicode(t *testing.T) {
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
pullOrSkip(ctx, t, client, req.Model)
- slog.Info("loading", "model", req.Model)
- err := client.Generate(ctx, &api.GenerateRequest{Model: req.Model}, func(response api.GenerateResponse) error { return nil })
- if err != nil {
- t.Fatalf("failed to load model %s: %s", req.Model, err)
- }
+ preloadGenerateModel(ctx, t, client, api.GenerateRequest{Model: req.Model})
defer func() {
// best effort unload once we're done with the model
client.Generate(ctx, &api.GenerateRequest{Model: req.Model, KeepAlive: &api.Duration{Duration: 0}}, func(rsp api.GenerateResponse) error { return nil })
@@ -81,15 +77,15 @@ func TestUnicode(t *testing.T) {
}, 180*time.Second, 30*time.Second)
}
-func TestExtendedUnicodeOutput(t *testing.T) {
+func runExtendedUnicodeOutput(t *testing.T, model string) {
if testModel != "" {
t.Skip("uses hardcoded model, not applicable with model override")
}
- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute)
defer cancel()
// Set up the test data
req := api.ChatRequest{
- Model: "gemma2:2b",
+ Model: model,
Messages: []api.Message{
{
Role: "user",
@@ -108,14 +104,14 @@ func TestExtendedUnicodeOutput(t *testing.T) {
DoChat(ctx, t, client, req, []string{"😀", "😊", "😁", "😂", "😄", "😃"}, 120*time.Second, 120*time.Second)
}
-func TestUnicodeModelDir(t *testing.T) {
+func runUnicodeModelDir(t *testing.T) {
// This is only useful for Windows with utf-16 characters, so skip this test for other platforms
if runtime.GOOS != "windows" {
t.Skip("Unicode test only applicable to windows")
}
// Only works for local testing
if os.Getenv("OLLAMA_TEST_EXISTING") != "" {
- t.Skip("TestUnicodeModelDir only works for local testing, skipping")
+ t.Skip("runUnicodeModelDir only works for local testing, skipping")
}
modelDir, err := os.MkdirTemp("", "ollama_埃")
@@ -127,7 +123,7 @@ func TestUnicodeModelDir(t *testing.T) {
t.Setenv("OLLAMA_MODELS", modelDir)
- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute)
defer cancel()
req := api.ChatRequest{
@@ -147,22 +143,22 @@ func TestUnicodeModelDir(t *testing.T) {
ChatTestHelper(ctx, t, req, blueSkyExpected)
}
-// TestNumPredict verifies that when num_predict is set, the model generates
+// runNumPredict verifies that when num_predict is set, the model generates
// exactly that many tokens. It uses logprobs to count the actual tokens output.
-func TestNumPredict(t *testing.T) {
+func runNumPredict(t *testing.T, model string) {
if testModel != "" {
t.Skip("uses hardcoded model, not applicable with model override")
}
- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute)
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
- pullOrSkip(ctx, t, client, "qwen3:0.6b")
+ pullOrSkip(ctx, t, client, model)
req := api.GenerateRequest{
- Model: "qwen3:0.6b",
+ Model: model,
Prompt: "Write a long story.",
Stream: &stream,
Logprobs: true,
diff --git a/integration/chat_cases_test.go b/integration/chat_cases_test.go
new file mode 100644
index 000000000..64805aeba
--- /dev/null
+++ b/integration/chat_cases_test.go
@@ -0,0 +1,144 @@
+//go:build integration
+
+package integration
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "os"
+ "strconv"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/format"
+)
+
+var sweepVRAMWarning sync.Once
+
+func registerChatCases(models []string) {
+ registerModelIntegrationCases("chat", models, runChatModel)
+}
+
+func runChatModel(t *testing.T, model string) {
+ t.Helper()
+
+ softTimeout, hardTimeout := getTimeouts(t)
+ slog.Info("Setting timeouts", "soft", softTimeout, "hard", hardTimeout)
+ ctx, cancel := context.WithTimeout(context.Background(), hardTimeout)
+ defer cancel()
+ client, _, cleanup := InitServerConnection(ctx, t)
+ defer cleanup()
+
+ if time.Since(started) > softTimeout {
+ t.Skip("skipping remaining tests to avoid excessive runtime")
+ }
+ skipRegisteredMinVRAM(t, model)
+ requireCapability(ctx, t, client, model, "completion")
+ skipIfTargetArchitecture(ctx, t, client, model)
+ skipIfModelTooLargeForSweepVRAM(ctx, t, client, model)
+
+ initialTimeout := 120 * time.Second
+ streamTimeout := 30 * time.Second
+ preloadGenerateModel(ctx, t, client, api.GenerateRequest{Model: model, KeepAlive: &api.Duration{Duration: 10 * time.Second}})
+ defer func() {
+ client.Generate(ctx, &api.GenerateRequest{Model: model, KeepAlive: &api.Duration{Duration: 0}}, func(rsp api.GenerateResponse) error { return nil })
+ }()
+
+ gpuPercent := getGPUPercent(ctx, t, client, model)
+ if gpuPercent < 80 {
+ slog.Warn("Low GPU percentage - increasing timeouts", "percent", gpuPercent)
+ initialTimeout = 240 * time.Second
+ streamTimeout = 40 * time.Second
+ }
+
+ req, anyResp := chatModelRequest(model)
+ DoChat(ctx, t, client, req, anyResp, initialTimeout, streamTimeout)
+}
+
+func chatModelRequest(model string) (api.ChatRequest, []string) {
+ req := api.ChatRequest{
+ Model: model,
+ Messages: []api.Message{
+ {
+ Role: "user",
+ Content: blueSkyPrompt,
+ },
+ },
+ KeepAlive: &api.Duration{Duration: 10 * time.Second},
+ Options: map[string]any{
+ "temperature": 0.1,
+ "seed": 123,
+ },
+ }
+ anyResp := blueSkyExpected
+
+ // Special cases
+ if model == "duckdb-nsql" {
+ anyResp = []string{"select", "from"}
+ } else if model == "granite3-guardian" || model == "shieldgemma" || model == "llama-guard3" || model == "bespoke-minicheck" {
+ anyResp = []string{"yes", "no", "safe", "unsafe"}
+ } else if model == "openthinker" {
+ anyResp = []string{"plugin", "im_sep", "components", "function call"}
+ } else if model == "starcoder" || model == "starcoder2" || model == "magicoder" || model == "deepseek-coder" {
+ req.Messages[0].Content = "def fibonacci():"
+ anyResp = []string{"f(n)", "sequence", "n-1", "main()", "__main__", "while"}
+ }
+
+ return req, anyResp
+}
+
+func skipIfTargetArchitecture(ctx context.Context, t *testing.T, client *api.Client, model string) {
+ t.Helper()
+
+ targetArch := os.Getenv("OLLAMA_TEST_ARCHITECTURE")
+ if targetArch == "" {
+ return
+ }
+
+ resp, err := client.Show(ctx, &api.ShowRequest{Name: model})
+ if err != nil {
+ t.Fatalf("unable to show model: %s", err)
+ }
+ arch := resp.ModelInfo["general.architecture"].(string)
+ if arch != targetArch {
+ t.Skip(fmt.Sprintf("Skipping %s architecture %s != %s", model, arch, targetArch))
+ }
+}
+
+func skipIfModelTooLargeForSweepVRAM(ctx context.Context, t *testing.T, client *api.Client, model string) {
+ t.Helper()
+
+ s := os.Getenv("OLLAMA_MAX_VRAM")
+ if s == "" {
+ sweepVRAMWarning.Do(func() {
+ slog.Warn("No VRAM info available, testing all models, so larger ones might timeout...")
+ })
+ return
+ }
+
+ maxVram, err := strconv.ParseUint(s, 10, 64)
+ if err != nil {
+ t.Fatalf("invalid OLLAMA_MAX_VRAM %v", err)
+ }
+
+ resp, err := client.List(ctx)
+ if err != nil {
+ t.Fatalf("list models failed %v", err)
+ }
+ for _, m := range resp.Models {
+ if modelNameMatches(model, m.Name) && float32(m.Size)*1.2 > float32(maxVram) {
+ t.Skipf("model %s is too large for available VRAM: %s > %s", model, format.HumanBytes(m.Size), format.HumanBytes(int64(maxVram)))
+ }
+ }
+}
+
+func modelNameMatches(model, name string) bool {
+ if name == model {
+ return true
+ }
+ return !strings.Contains(model, ":") && strings.HasPrefix(name, model+":")
+}
diff --git a/integration/concurrency_test.go b/integration/concurrency_test.go
index 8428fe462..a5097f350 100644
--- a/integration/concurrency_test.go
+++ b/integration/concurrency_test.go
@@ -20,7 +20,7 @@ import (
)
// Send multiple requests in parallel (concurrently) to a single model and ensure responses are expected
-func TestConcurrentChat(t *testing.T) {
+func runConcurrentChat(t *testing.T) {
// Assumes all requests have the same model
req, resp := ChatRequests()
numParallel := int(envconfig.NumParallel() + 1)
@@ -31,16 +31,10 @@ func TestConcurrentChat(t *testing.T) {
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
+ pullOrSkip(ctx, t, client, req[0].Model)
// Get the server running (if applicable) warm the model up with a single initial request
- slog.Info("loading", "model", req[0].Model)
- err := client.Generate(ctx,
- &api.GenerateRequest{Model: req[0].Model, KeepAlive: &api.Duration{Duration: 10 * time.Second}},
- func(response api.GenerateResponse) error { return nil },
- )
- if err != nil {
- t.Fatalf("failed to load model %s: %s", req[0].Model, err)
- }
+ preloadGenerateModel(ctx, t, client, api.GenerateRequest{Model: req[0].Model, KeepAlive: &api.Duration{Duration: 10 * time.Second}})
var wg sync.WaitGroup
r := rand.New(rand.NewSource(0))
@@ -66,7 +60,7 @@ func TestConcurrentChat(t *testing.T) {
// Stress the scheduler and attempt to load more models than will fit to cause thrashing
// This test will always load at least 2 models even on CPU based systems
-func TestMultiModelStress(t *testing.T) {
+func runMultiModelStress(t *testing.T) {
if testModel != "" {
t.Skip("uses hardcoded models, not applicable with model override")
}
@@ -85,7 +79,6 @@ func TestMultiModelStress(t *testing.T) {
"llama3.2:1b",
"qwen3:0.6b",
"gemma2:2b",
- "deepseek-r1:1.5b", // qwen2 arch
"gemma3:270m",
}
mediumModels := []string{
@@ -126,12 +119,8 @@ func TestMultiModelStress(t *testing.T) {
slog.Info("Loading models to find how many can fit in VRAM before overflowing")
chooseModels:
for i, model := range chosenModels {
- req := &api.GenerateRequest{Model: model} // Leave KeepAlive unset so they stay loaded until the scheduler decides to unload them
- slog.Info("loading", "model", model)
- err = client.Generate(ctx, req, func(response api.GenerateResponse) error { return nil })
- if err != nil {
- t.Fatalf("failed to load model %s: %s", model, err)
- }
+ // Leave KeepAlive unset so they stay loaded until the scheduler decides to unload them.
+ preloadGenerateModel(ctx, t, client, api.GenerateRequest{Model: model})
targetLoadCount++
if i > 0 {
models, err := client.ListRunning(ctx)
diff --git a/integration/context_test.go b/integration/context_test.go
index 82f932997..a789ce01d 100644
--- a/integration/context_test.go
+++ b/integration/context_test.go
@@ -14,7 +14,12 @@ import (
"github.com/ollama/ollama/api"
)
-func TestLongInputContext(t *testing.T) {
+const (
+ longInputTimeout = 2 * time.Minute
+ longInputModelOverrideTimeout = 3 * time.Minute
+)
+
+func runLongInputContext(t *testing.T) {
// Setting NUM_PARALLEL to 1 ensures the allocated context is exactly what
// we asked for and there is nothing extra that we could spill over into.
// Context shift happens after a prompt has been admitted to a slot. Initial
@@ -23,7 +28,11 @@ func TestLongInputContext(t *testing.T) {
// prompt while llama-server reports it as too large to admit.
t.Setenv("OLLAMA_NUM_PARALLEL", "1")
- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ timeout := longInputTimeout
+ if testModel != "" {
+ timeout = longInputModelOverrideTimeout
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
req := api.ChatRequest{
Model: smol,
@@ -79,7 +88,7 @@ func isContextLimitError(err string) bool {
strings.Contains(err, "too long"))
}
-func TestContextExhaustion(t *testing.T) {
+func runContextExhaustion(t *testing.T) {
// Setting NUM_PARALLEL to 1 ensures the allocated context is exactly what
// we asked for and there is nothing extra that we could spill over into
t.Setenv("OLLAMA_NUM_PARALLEL", "1")
@@ -128,11 +137,13 @@ func containsEmoji(s string) bool {
}
// Send multiple generate requests with prior context and ensure the response is coherant and expected
-func TestParallelGenerateWithHistory(t *testing.T) {
+func runParallelGenerateWithHistory(t *testing.T, modelName string) {
if testModel != "" {
- t.Skip("uses hardcoded model, not applicable with model override")
+ // The Generate API's Context field (token array continuation) is not
+ // supported by all runners (e.g. MLX). Chat history works; this is
+ // the only generate-specific continuation path.
+ t.Skip("generate context continuation not supported by all runners")
}
- modelName := "gpt-oss:20b"
req, resp := GenerateRequests()
numParallel := 2
iterLimit := 2
@@ -144,16 +155,10 @@ func TestParallelGenerateWithHistory(t *testing.T) {
defer cleanup()
initialTimeout := 120 * time.Second
streamTimeout := 20 * time.Second
+ prepareParallelHistoryModel(ctx, t, client, modelName)
// Get the server running (if applicable) warm the model up with a single initial request
- slog.Info("loading", "model", modelName)
- err := client.Generate(ctx,
- &api.GenerateRequest{Model: modelName, KeepAlive: &api.Duration{Duration: 10 * time.Second}},
- func(response api.GenerateResponse) error { return nil },
- )
- if err != nil {
- t.Fatalf("failed to load model %s: %s", modelName, err)
- }
+ preloadGenerateModel(ctx, t, client, api.GenerateRequest{Model: modelName, KeepAlive: &api.Duration{Duration: 10 * time.Second}})
gpuPercent := getGPUPercent(ctx, t, client, modelName)
if gpuPercent < 80 && gpuPercent > 50 {
slog.Warn("Low GPU percentage - increasing timeouts", "percent", gpuPercent)
@@ -190,7 +195,7 @@ func TestParallelGenerateWithHistory(t *testing.T) {
}
// Send generate requests with prior context and ensure the response is coherant and expected
-func TestGenerateWithHistory(t *testing.T) {
+func runGenerateWithHistory(t *testing.T) {
if testModel != "" {
// The Generate API's Context field (token array continuation) is not
// supported by all runners (e.g. MLX). Chat history works; this is
@@ -212,16 +217,10 @@ func TestGenerateWithHistory(t *testing.T) {
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
+ pullOrSkip(ctx, t, client, req.Model)
// Get the server running (if applicable) warm the model up with a single initial request
- slog.Info("loading", "model", req.Model)
- err := client.Generate(ctx,
- &api.GenerateRequest{Model: req.Model, KeepAlive: &api.Duration{Duration: 10 * time.Second}, Options: req.Options},
- func(response api.GenerateResponse) error { return nil },
- )
- if err != nil {
- t.Fatalf("failed to load model %s: %s", req.Model, err)
- }
+ preloadGenerateModel(ctx, t, client, api.GenerateRequest{Model: req.Model, KeepAlive: &api.Duration{Duration: 10 * time.Second}, Options: req.Options})
req.Context = DoGenerate(ctx, t, client, req, rainbowExpected, 30*time.Second, 20*time.Second)
@@ -236,11 +235,7 @@ func TestGenerateWithHistory(t *testing.T) {
}
// Send multiple chat requests with prior context and ensure the response is coherant and expected
-func TestParallelChatWithHistory(t *testing.T) {
- if testModel != "" {
- t.Skip("uses hardcoded model, not applicable with model override")
- }
- modelName := "gpt-oss:20b"
+func runParallelChatWithHistory(t *testing.T, modelName string) {
req, resp := ChatRequests()
numParallel := 2
iterLimit := 2
@@ -252,16 +247,10 @@ func TestParallelChatWithHistory(t *testing.T) {
defer cleanup()
initialTimeout := 120 * time.Second
streamTimeout := 20 * time.Second
+ prepareParallelHistoryModel(ctx, t, client, modelName)
// Get the server running (if applicable) warm the model up with a single initial empty request
- slog.Info("loading", "model", modelName)
- err := client.Generate(ctx,
- &api.GenerateRequest{Model: modelName, KeepAlive: &api.Duration{Duration: 10 * time.Second}},
- func(response api.GenerateResponse) error { return nil },
- )
- if err != nil {
- t.Fatalf("failed to load model %s: %s", modelName, err)
- }
+ preloadGenerateModel(ctx, t, client, api.GenerateRequest{Model: modelName, KeepAlive: &api.Duration{Duration: 10 * time.Second}})
gpuPercent := getGPUPercent(ctx, t, client, modelName)
if gpuPercent < 80 && gpuPercent > 50 {
slog.Warn("Low GPU percentage - increasing timeouts", "percent", gpuPercent)
@@ -302,8 +291,16 @@ func TestParallelChatWithHistory(t *testing.T) {
wg.Wait()
}
+func prepareParallelHistoryModel(ctx context.Context, t *testing.T, client *api.Client, modelName string) {
+ t.Helper()
+ skipRegisteredMinVRAM(t, modelName)
+ requireCapability(ctx, t, client, modelName, "completion")
+ skipIfTargetArchitecture(ctx, t, client, modelName)
+ skipIfModelTooLargeForSweepVRAM(ctx, t, client, modelName)
+}
+
// Send generate requests with prior context and ensure the response is coherant and expected
-func TestChatWithHistory(t *testing.T) {
+func runChatWithHistory(t *testing.T) {
req := api.ChatRequest{
Model: smol,
Stream: &stream,
@@ -324,16 +321,10 @@ func TestChatWithHistory(t *testing.T) {
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
+ pullOrSkip(ctx, t, client, req.Model)
// Get the server running (if applicable) warm the model up with a single initial request
- slog.Info("loading", "model", req.Model)
- err := client.Generate(ctx,
- &api.GenerateRequest{Model: req.Model, KeepAlive: &api.Duration{Duration: 10 * time.Second}, Options: req.Options},
- func(response api.GenerateResponse) error { return nil },
- )
- if err != nil {
- t.Fatalf("failed to load model %s: %s", req.Model, err)
- }
+ preloadGenerateModel(ctx, t, client, api.GenerateRequest{Model: req.Model, KeepAlive: &api.Duration{Duration: 10 * time.Second}, Options: req.Options})
assistant := DoChat(ctx, t, client, req, rainbowExpected, 30*time.Second, 20*time.Second)
diff --git a/integration/create_test.go b/integration/create_test.go
index cca2d4270..3cd5e8129 100644
--- a/integration/create_test.go
+++ b/integration/create_test.go
@@ -136,7 +136,7 @@ func runOllamaCreate(ctx context.Context, t *testing.T, args ...string) {
}
}
-func TestCreateSafetensorsLLM(t *testing.T) {
+func runCreateSafetensorsLLM(t *testing.T) {
if testModel != "" {
t.Skip("exercises create pipeline with a fixed source model, not applicable with model override")
}
@@ -214,7 +214,7 @@ func TestCreateSafetensorsLLM(t *testing.T) {
}
}
-func TestCreateGGUF(t *testing.T) {
+func runCreateGGUF(t *testing.T) {
if testModel != "" {
t.Skip("exercises create pipeline with a fixed source model, not applicable with model override")
}
diff --git a/integration/embed_cases_test.go b/integration/embed_cases_test.go
new file mode 100644
index 000000000..51d323770
--- /dev/null
+++ b/integration/embed_cases_test.go
@@ -0,0 +1,204 @@
+//go:build integration
+
+package integration
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/ollama/ollama/api"
+)
+
+func registerEmbeddingCases(models []string) {
+ registerEmbeddingCasesWithFallback(models, false)
+}
+
+func registerLibraryEmbeddingCases(models []string) {
+ registerEmbeddingCasesWithFallback(models, true)
+}
+
+func registerEmbeddingCasesWithFallback(models []string, smokeMissing bool) {
+ testCases, err := loadEmbeddingTestCases()
+ if err != nil {
+ registerIntegrationCases(integrationCase{
+ Key: "embed/testdata",
+ Case: "embed",
+ Model: "testdata",
+ Run: func(t *testing.T) {
+ t.Fatalf("failed to load embedding test data: %s", err)
+ },
+ })
+ return
+ }
+
+ if testModel != "" {
+ models = []string{testModel}
+ }
+
+ cases := make([]integrationCase, 0, len(models))
+ for _, model := range models {
+ model := model
+ expected, ok := embeddingExpected(testCases, model)
+ if !ok {
+ if smokeMissing || testModel != "" {
+ cases = append(cases, embeddingSmokeCase(model))
+ continue
+ }
+ cases = append(cases, integrationCase{
+ Key: "embed/" + model,
+ Case: "embed",
+ Model: model,
+ Run: func(t *testing.T) {
+ t.Skipf("no embedding expectation for model %s", model)
+ },
+ })
+ continue
+ }
+
+ cases = append(cases, embeddingCase(model, expected))
+ }
+ registerIntegrationCases(cases...)
+}
+
+func embeddingSmokeCase(model string) integrationCase {
+ return integrationCase{
+ Key: "embed/" + model,
+ Case: "embed",
+ Model: model,
+ Run: func(t *testing.T) {
+ runEmbeddingSmokeModel(t, model)
+ },
+ }
+}
+
+func embeddingCase(model string, expected []float64) integrationCase {
+ return integrationCase{
+ Key: "embed/" + model,
+ Case: "embed",
+ Model: model,
+ Run: func(t *testing.T) {
+ runEmbeddingModel(t, model, expected)
+ },
+ }
+}
+
+func loadEmbeddingTestCases() (map[string][]float64, error) {
+ data, err := os.ReadFile(filepath.Join("testdata", "embed.json"))
+ if err != nil {
+ return nil, err
+ }
+ testCases := map[string][]float64{}
+ if err := json.Unmarshal(data, &testCases); err != nil {
+ return nil, err
+ }
+ return testCases, nil
+}
+
+func embeddingExpected(testCases map[string][]float64, model string) ([]float64, bool) {
+ if expected, ok := testCases[model]; ok {
+ return expected, true
+ }
+ if !strings.Contains(model, ":") {
+ expected, ok := testCases[model+":latest"]
+ return expected, ok
+ }
+ return nil, false
+}
+
+func runEmbeddingModel(t *testing.T, model string, expected []float64) {
+ t.Helper()
+
+ softTimeout, hardTimeout := getTimeouts(t)
+ ctx, cancel := context.WithTimeout(context.Background(), hardTimeout)
+ defer cancel()
+ client, _, cleanup := InitServerConnection(ctx, t)
+ defer cleanup()
+
+ if time.Since(started) > softTimeout {
+ t.Skip("skipping remaining tests to avoid excessive runtime")
+ }
+ pullOrSkip(ctx, t, client, model)
+ skipIfModelTooLargeForSweepVRAM(ctx, t, client, model)
+
+ req := api.EmbeddingRequest{
+ Model: model,
+ Prompt: "why is the sky blue?",
+ KeepAlive: &api.Duration{Duration: 10 * time.Second},
+ Options: map[string]any{
+ "temperature": 0,
+ "seed": 123,
+ },
+ }
+ resp, err := client.Embeddings(ctx, &req)
+ if err != nil {
+ t.Fatalf("embeddings call failed %s", err)
+ }
+ defer func() {
+ client.Generate(ctx, &api.GenerateRequest{Model: req.Model, KeepAlive: &api.Duration{Duration: 0}}, func(rsp api.GenerateResponse) error { return nil })
+ }()
+ if len(resp.Embedding) == 0 {
+ t.Errorf("zero length embedding response")
+ }
+ if len(expected) != len(resp.Embedding) {
+ expStr := make([]string, len(resp.Embedding))
+ for i, v := range resp.Embedding {
+ expStr[i] = fmt.Sprintf("%0.6f", v)
+ }
+ // When adding new models, use this output to populate the testdata/embed.json
+ fmt.Printf("expected\n%s\n", strings.Join(expStr, ", "))
+ t.Fatalf("expected %d, got %d", len(expected), len(resp.Embedding))
+ }
+ sim := cosineSimilarity(resp.Embedding, expected)
+ if sim < 0.99 {
+ t.Fatalf("expected %v, got %v (similarity: %f)", expected[0:5], resp.Embedding[0:5], sim)
+ }
+}
+
+func runEmbeddingSmokeModel(t *testing.T, model string) {
+ t.Helper()
+
+ softTimeout, hardTimeout := getTimeouts(t)
+ ctx, cancel := context.WithTimeout(context.Background(), hardTimeout)
+ defer cancel()
+ client, _, cleanup := InitServerConnection(ctx, t)
+ defer cleanup()
+
+ if time.Since(started) > softTimeout {
+ t.Skip("skipping remaining tests to avoid excessive runtime")
+ }
+ requireCapability(ctx, t, client, model, "embedding")
+ skipIfModelTooLargeForSweepVRAM(ctx, t, client, model)
+
+ req := api.EmbedRequest{
+ Model: model,
+ Input: []string{"cat", "kitten", "dog"},
+ KeepAlive: &api.Duration{Duration: 10 * time.Second},
+ }
+ resp, err := embedTestHelper(ctx, client, t, req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() {
+ client.Generate(ctx, &api.GenerateRequest{Model: req.Model, KeepAlive: &api.Duration{Duration: 0}}, func(rsp api.GenerateResponse) error { return nil })
+ }()
+ if len(resp.Embeddings) != 3 {
+ t.Fatalf("expected 3 embeddings, got %d", len(resp.Embeddings))
+ }
+ for i, embedding := range resp.Embeddings {
+ if len(embedding) == 0 {
+ t.Fatalf("embedding %d was empty", i)
+ }
+ }
+
+ cosRelated := cosineSimilarity(resp.Embeddings[0], resp.Embeddings[1])
+ cosUnrelated := cosineSimilarity(resp.Embeddings[0], resp.Embeddings[2])
+ if cosRelated <= cosUnrelated {
+ t.Fatalf("expected related terms to be closer than unrelated terms: cat/kitten=%f cat/dog=%f", cosRelated, cosUnrelated)
+ }
+}
diff --git a/integration/embed_test.go b/integration/embed_test.go
index ea32b5b35..31f34cd39 100644
--- a/integration/embed_test.go
+++ b/integration/embed_test.go
@@ -10,7 +10,6 @@ import (
"testing"
"time"
- "github.com/google/go-cmp/cmp"
"github.com/ollama/ollama/api"
)
@@ -61,6 +60,19 @@ func requireEmbedErrorContainsAny(t *testing.T, err error, substrings ...string)
t.Fatalf("expected error containing one of %q, got: %v", substrings, err)
}
+func requireSimilarEmbedding(t *testing.T, want, got []float32) {
+ t.Helper()
+
+ if len(got) != len(want) {
+ t.Fatalf("expected %d embedding floats, got %d", len(want), len(got))
+ }
+
+ sim := cosineSimilarity(got, want)
+ if sim < 0.999 {
+ t.Fatalf("expected embedding similar to %v, got %v (similarity: %f)", want[0:5], got[0:5], sim)
+ }
+}
+
func euclideanDistance[V float32 | float64](v1, v2 []V) V {
if len(v1) != len(v2) {
return V(math.Inf(1))
@@ -88,13 +100,13 @@ func manhattanDistance[V float32 | float64](v1, v2 []V) V {
return sum
}
-func TestEmbedCosineDistanceCorrelation(t *testing.T) {
+func runEmbedCosineDistanceCorrelation(t *testing.T, models []string) {
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute)
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
- for _, model := range testModels(libraryEmbedModels) {
+ for _, model := range testModels(models) {
t.Run(model, func(t *testing.T) {
if testModel != "" {
requireCapability(ctx, t, client, model, "embedding")
@@ -163,7 +175,7 @@ func TestEmbedCosineDistanceCorrelation(t *testing.T) {
}
}
-func TestAllMiniLMEmbeddings(t *testing.T) {
+func runAllMiniLMEmbeddings(t *testing.T) {
if testModel != "" {
t.Skip("uses hardcoded model, not applicable with model override")
}
@@ -196,7 +208,7 @@ func TestAllMiniLMEmbeddings(t *testing.T) {
}
}
-func TestAllMiniLMEmbed(t *testing.T) {
+func runAllMiniLMEmbed(t *testing.T) {
if testModel != "" {
t.Skip("uses hardcoded model, not applicable with model override")
}
@@ -236,7 +248,7 @@ func TestAllMiniLMEmbed(t *testing.T) {
}
}
-func TestAllMiniLMBatchEmbed(t *testing.T) {
+func runAllMiniLMBatchEmbed(t *testing.T) {
if testModel != "" {
t.Skip("uses hardcoded model, not applicable with model override")
}
@@ -286,7 +298,7 @@ func TestAllMiniLMBatchEmbed(t *testing.T) {
}
}
-func TestAllMiniLMEmbedTruncate(t *testing.T) {
+func runAllMiniLMEmbedTruncate(t *testing.T) {
if testModel != "" {
t.Skip("uses hardcoded model, not applicable with model override")
}
@@ -321,9 +333,7 @@ func TestAllMiniLMEmbedTruncate(t *testing.T) {
t.Fatal(err)
}
- if diff := cmp.Diff(want.Embeddings[0], got.Embeddings[0]); diff != "" {
- t.Errorf("embedding mismatch (-want +got):\n%s", diff)
- }
+ requireSimilarEmbedding(t, want.Embeddings[0], got.Embeddings[0])
},
},
{
@@ -338,9 +348,7 @@ func TestAllMiniLMEmbedTruncate(t *testing.T) {
t.Fatal(err)
}
t.Logf("PromptEvalCount: want=%d got=%d", want.PromptEvalCount, got.PromptEvalCount)
- if diff := cmp.Diff(want.Embeddings[0], got.Embeddings[0]); diff != "" {
- t.Errorf("embedding mismatch (-want +got):\n%s", diff)
- }
+ requireSimilarEmbedding(t, want.Embeddings[0], got.Embeddings[0])
},
},
{
@@ -356,9 +364,7 @@ func TestAllMiniLMEmbedTruncate(t *testing.T) {
t.Fatal(err)
}
t.Logf("PromptEvalCount: want=%d got=%d", want.PromptEvalCount, got.PromptEvalCount)
- if diff := cmp.Diff(want.Embeddings[0], got.Embeddings[0]); diff != "" {
- t.Errorf("embedding mismatch (-want +got):\n%s", diff)
- }
+ requireSimilarEmbedding(t, want.Embeddings[0], got.Embeddings[0])
},
},
{
@@ -432,7 +438,7 @@ func embedTestHelper(ctx context.Context, client *api.Client, t *testing.T, req
return client.Embed(ctx, &req)
}
-func TestEmbedTruncation(t *testing.T) {
+func runEmbedTruncation(t *testing.T, models []string) {
// Use test deadline if set, otherwise default to 2 minutes
timeout := 2 * time.Minute
if deadline, ok := t.Deadline(); ok {
@@ -443,7 +449,7 @@ func TestEmbedTruncation(t *testing.T) {
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
- for _, model := range testModels(libraryEmbedModels) {
+ for _, model := range testModels(models) {
model := model
t.Run(model, func(t *testing.T) {
if testModel != "" {
@@ -454,6 +460,9 @@ func TestEmbedTruncation(t *testing.T) {
t.Skip("skipping remaining tests to avoid timeout")
}
+ pullOrSkip(ctx, t, client, model)
+ skipIfModelTooLargeForSweepVRAM(ctx, t, client, model)
+
// Give each model its own budget to account for first-time pulls/loads
mctx, mcancel := context.WithTimeout(ctx, 3*time.Minute)
defer mcancel()
@@ -507,19 +516,22 @@ func TestEmbedTruncation(t *testing.T) {
}
}
-// TestEmbedLargeInput tests that embedding models can handle large inputs that would exceed typical batch sizes.
-func TestEmbedLargeInput(t *testing.T) {
+// runEmbedLargeInput tests that embedding models can handle large inputs that would exceed typical batch sizes.
+func runEmbedLargeInput(t *testing.T, models []string) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
- for _, model := range testModels(libraryEmbedModels) {
+ for _, model := range testModels(models) {
model := model
t.Run(model, func(t *testing.T) {
if testModel != "" {
requireCapability(ctx, t, client, model, "embedding")
}
+ pullOrSkip(ctx, t, client, model)
+ skipIfModelTooLargeForSweepVRAM(ctx, t, client, model)
+
mctx, mcancel := context.WithTimeout(ctx, 2*time.Minute)
defer mcancel()
@@ -567,11 +579,11 @@ func TestEmbedLargeInput(t *testing.T) {
}
}
-// TestEmbedStatusCode tests that errors from the embedding endpoint
+// runEmbedStatusCode tests that errors from the embedding endpoint
// properly preserve their HTTP status codes when returned to the client.
// This test specifically checks the error handling path in EmbedHandler
// where api.StatusError errors should maintain their original status code.
-func TestEmbedStatusCode(t *testing.T) {
+func runEmbedStatusCode(t *testing.T, models []string) {
// Use test deadline if set, otherwise default to 2 minutes
timeout := 2 * time.Minute
if deadline, ok := t.Deadline(); ok {
@@ -582,7 +594,7 @@ func TestEmbedStatusCode(t *testing.T) {
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
- for _, model := range testModels(libraryEmbedModels) {
+ for _, model := range testModels(models) {
model := model
t.Run(model, func(t *testing.T) {
if testModel != "" {
@@ -598,6 +610,7 @@ func TestEmbedStatusCode(t *testing.T) {
// Pull the model if needed
pullOrSkip(mctx, t, client, model)
+ skipIfModelTooLargeForSweepVRAM(mctx, t, client, model)
t.Run("truncation error status code", func(t *testing.T) {
truncFalse := false
diff --git a/integration/imagegen_test.go b/integration/imagegen_test.go
index 6b598b844..29d64bce2 100644
--- a/integration/imagegen_test.go
+++ b/integration/imagegen_test.go
@@ -13,7 +13,7 @@ import (
"github.com/ollama/ollama/api"
)
-func TestImageGeneration(t *testing.T) {
+func runImageGeneration(t *testing.T) {
if testModel != "" {
t.Skip("uses hardcoded models, not applicable with model override")
}
@@ -51,6 +51,7 @@ func TestImageGeneration(t *testing.T) {
t.Logf("Generating image with prompt: %s", tc.prompt)
imageBase64, err := generateImage(ctx, client, tc.imageGenModel, tc.prompt)
if err != nil {
+ skipIfMLXUnsupported(t, err)
if strings.Contains(err.Error(), "image generation not available") {
t.Skip("Target system does not support image generation")
} else if strings.Contains(err.Error(), "executable file not found in") { // Windows pattern, not yet supported
@@ -79,10 +80,7 @@ func TestImageGeneration(t *testing.T) {
t.Logf("Generated image: %d bytes", len(imageData))
// Preload vision model and check GPU loading
- err = client.Generate(ctx, &api.GenerateRequest{Model: tc.visionModel}, func(response api.GenerateResponse) error { return nil })
- if err != nil {
- t.Fatalf("failed to load vision model: %v", err)
- }
+ preloadGenerateModel(ctx, t, client, api.GenerateRequest{Model: tc.visionModel})
// Use vision model to describe the image
chatReq := api.ChatRequest{
diff --git a/integration/library_models_test.go b/integration/library_models_test.go
deleted file mode 100644
index 536082ab8..000000000
--- a/integration/library_models_test.go
+++ /dev/null
@@ -1,72 +0,0 @@
-//go:build integration && library
-
-package integration
-
-import (
- "context"
- "fmt"
- "log/slog"
- "os"
- "testing"
- "time"
-
- "github.com/ollama/ollama/api"
-)
-
-// First run of this scenario on a target system will take a long time to download
-// ~1.5TB of models. Set a sufficiently large -timeout for your network speed
-func TestLibraryModelsChat(t *testing.T) {
- softTimeout, hardTimeout := getTimeouts(t)
- slog.Info("Setting timeouts", "soft", softTimeout, "hard", hardTimeout)
- ctx, cancel := context.WithTimeout(context.Background(), hardTimeout)
- defer cancel()
- client, _, cleanup := InitServerConnection(ctx, t)
- defer cleanup()
- targetArch := os.Getenv("OLLAMA_TEST_ARCHITECTURE")
-
- for _, model := range testModels(libraryChatModels) {
- t.Run(model, func(t *testing.T) {
- if time.Now().Sub(started) > softTimeout {
- t.Skip("skipping remaining tests to avoid excessive runtime")
- }
- pullOrSkip(ctx, t, client, model)
- if targetArch != "" {
- resp, err := client.Show(ctx, &api.ShowRequest{Name: model})
- if err != nil {
- t.Fatalf("unable to show model: %s", err)
- }
- arch := resp.ModelInfo["general.architecture"].(string)
- if arch != targetArch {
- t.Skip(fmt.Sprintf("Skipping %s architecture %s != %s", model, arch, targetArch))
- }
- }
- req := api.ChatRequest{
- Model: model,
- Messages: []api.Message{
- {
- Role: "user",
- Content: blueSkyPrompt,
- },
- },
- KeepAlive: &api.Duration{Duration: 10 * time.Second},
- Options: map[string]interface{}{
- "temperature": 0.1,
- "seed": 123,
- },
- }
- anyResp := blueSkyExpected
- // Special cases
- if model == "duckdb-nsql" {
- anyResp = []string{"select", "from"}
- } else if model == "granite3-guardian" || model == "shieldgemma" || model == "llama-guard3" || model == "bespoke-minicheck" {
- anyResp = []string{"yes", "no", "safe", "unsafe"}
- } else if model == "openthinker" {
- anyResp = []string{"plugin", "im_sep", "components", "function call"}
- } else if model == "starcoder" || model == "starcoder2" || model == "magicoder" || model == "deepseek-coder" {
- req.Messages[0].Content = "def fibonacci():"
- anyResp = []string{"f(n)", "sequence", "n-1", "main()", "__main__", "while"}
- }
- DoChat(ctx, t, client, req, anyResp, 120*time.Second, 30*time.Second)
- })
- }
-}
diff --git a/integration/llm_image_test.go b/integration/llm_image_test.go
index e35ceb57e..1bf621782 100644
--- a/integration/llm_image_test.go
+++ b/integration/llm_image_test.go
@@ -11,69 +11,53 @@ import (
"github.com/ollama/ollama/api"
)
-func TestVisionModels(t *testing.T) {
- skipUnderMinVRAM(t, 6)
-
- defaultVisionModels := []string{
- "gemma4",
- "qwen2.5vl",
- // "llama3.2-vision", // TODO: re-enable when llama.cpp supports mllama.
- "gemma3",
- "qwen3-vl:8b",
- "qwen3-vl:30b",
- "ministral-3",
- }
+func registerVisionTextCases(models []string) {
+ registerModelIntegrationCases("vision-text", models, runVisionTextModel)
+}
- skipIfNoVisionOverride(t)
+func runVisionTextModel(t *testing.T, model string) {
+ t.Helper()
- for _, model := range testModels(defaultVisionModels) {
- t.Run(model, func(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
- defer cancel()
- client, _, cleanup := InitServerConnection(ctx, t)
- defer cleanup()
+ skipUnderMinVRAM(t, 6)
+ skipKnownIntegrationFlake(t, "vision-text", model)
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+ client, _, cleanup := InitServerConnection(ctx, t)
+ defer cleanup()
- requireCapability(ctx, t, client, model, "vision")
- pullOrSkip(ctx, t, client, model)
+ requireCapability(ctx, t, client, model, "vision")
+ pullOrSkip(ctx, t, client, model)
- image, err := base64.StdEncoding.DecodeString(imageEncoding)
- if err != nil {
- t.Fatal(err)
- }
- req := api.ChatRequest{
- Model: model,
- Messages: []api.Message{
- {
- Role: "user",
- Content: "what does the text in this image say?",
- Images: []api.ImageData{
- image,
- },
- },
- },
- Stream: &stream,
- Options: map[string]any{
- "seed": 42,
- "temperature": 0.0,
+ image, err := base64.StdEncoding.DecodeString(imageEncoding)
+ if err != nil {
+ t.Fatal(err)
+ }
+ req := api.ChatRequest{
+ Model: model,
+ Messages: []api.Message{
+ {
+ Role: "user",
+ Content: "what does the text in this image say?",
+ Images: []api.ImageData{
+ image,
},
- KeepAlive: &api.Duration{Duration: 10 * time.Second},
- }
-
- // Preload to skip if we're less than 80% on GPU to avoid extremely slow tests
- err = client.Generate(ctx, &api.GenerateRequest{Model: req.Model}, func(response api.GenerateResponse) error { return nil })
- if err != nil {
- t.Fatalf("failed to load model %s: %s", req.Model, err)
- }
- skipIfNotGPULoaded(ctx, t, client, req.Model, 80)
-
- // Note: sometimes it returns "the ollamas" sometimes "the ollams"
- // llava models on CPU can be quite slow to start
- DoChat(ctx, t, client, req, []string{"the ollam"}, 240*time.Second, 30*time.Second)
- })
+ },
+ },
+ Stream: &stream,
+ Options: map[string]any{
+ "seed": 42,
+ "temperature": 0.0,
+ },
+ KeepAlive: &api.Duration{Duration: 10 * time.Second},
}
+
+ // Preload to skip if we're less than 80% on GPU to avoid extremely slow tests
+ preloadGenerateModel(ctx, t, client, api.GenerateRequest{Model: req.Model})
+ skipIfNotGPULoaded(ctx, t, client, req.Model, 80)
+ DoChat(ctx, t, client, req, []string{"the ollam", "ollamas"}, 240*time.Second, 30*time.Second)
}
-func TestIntegrationSplitBatch(t *testing.T) {
+func runIntegrationSplitBatch(t *testing.T, model string) {
if testModel != "" {
t.Skip("uses hardcoded model, not applicable with model override")
}
@@ -83,7 +67,7 @@ func TestIntegrationSplitBatch(t *testing.T) {
t.Fatal(err)
}
req := api.GenerateRequest{
- Model: "gemma3:4b",
+ Model: model,
// Fill up a chunk of the batch so the image will partially spill over into the next one
System: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed aliquet, justo in malesuada lobortis, odio ligula volutpat quam, quis faucibus ipsum magna quis sapien. Aliquam in venenatis diam, eu viverra magna. Phasellus imperdiet hendrerit volutpat. Vivamus sem ex, facilisis placerat felis non, dictum elementum est. Phasellus aliquam imperdiet lacus, eget placerat ligula sodales vel. Pellentesque nec auctor mi. Curabitur arcu nisi, faucibus eget nunc id, viverra interdum mi. Curabitur ornare ipsum ex, ac euismod ex aliquam in. Vestibulum id magna at purus accumsan fermentum. Proin scelerisque posuere nunc quis interdum. Maecenas sed mollis nisl. Etiam vitae ipsum interdum, placerat est quis, tincidunt velit. Nullam tempor nibh non lorem volutpat efficitur. Cras laoreet diam imperdiet ipsum auctor bibendum. Suspendisse ultrices urna sed metus sagittis suscipit. Quisque ullamcorper aliquam nibh ut mollis. Aenean dapibus mauris pharetra, venenatis elit ac, hendrerit odio. Cras vestibulum erat tempor, lobortis justo eu, lobortis ipsum. Nam laoreet dapibus sem. Proin vel diam ultrices, elementum ante et, ornare lectus. Proin eu accumsan nisl. Praesent ac ex vitae ipsum vulputate tristique facilisis sit amet lacus. Nullam faucibus magna a pellentesque pretium. Nunc lacinia ullamcorper sollicitudin. Donec vitae accumsan turpis, sed porttitor est. Donec porttitor mi vitae augue faucibus, vel mollis diam tincidunt.",
Prompt: "what does the text in this image say?",
diff --git a/integration/max_queue_test.go b/integration/max_queue_test.go
index a8daab93c..de7bc34d7 100644
--- a/integration/max_queue_test.go
+++ b/integration/max_queue_test.go
@@ -16,7 +16,7 @@ import (
"github.com/ollama/ollama/api"
)
-func TestMaxQueue(t *testing.T) {
+func runMaxQueue(t *testing.T) {
t.Skip("this test needs to be re-evaluated to use a proper embedding model")
if os.Getenv("OLLAMA_TEST_EXISTING") != "" {
diff --git a/integration/model_arch_test.go b/integration/model_arch_test.go
deleted file mode 100644
index 3bb27d10e..000000000
--- a/integration/model_arch_test.go
+++ /dev/null
@@ -1,187 +0,0 @@
-//go:build integration && models
-
-package integration
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "io/ioutil"
- "log/slog"
- "os"
- "path/filepath"
- "strconv"
- "strings"
- "testing"
- "time"
-
- "github.com/ollama/ollama/api"
- "github.com/ollama/ollama/format"
-)
-
-func TestModelsChat(t *testing.T) {
- softTimeout, hardTimeout := getTimeouts(t)
- slog.Info("Setting timeouts", "soft", softTimeout, "hard", hardTimeout)
- ctx, cancel := context.WithTimeout(context.Background(), hardTimeout)
- defer cancel()
- client, _, cleanup := InitServerConnection(ctx, t)
- defer cleanup()
-
- // TODO use info API eventually
- var maxVram uint64
- var err error
- if s := os.Getenv("OLLAMA_MAX_VRAM"); s != "" {
- maxVram, err = strconv.ParseUint(s, 10, 64)
- if err != nil {
- t.Fatalf("invalid OLLAMA_MAX_VRAM %v", err)
- }
- } else {
- slog.Warn("No VRAM info available, testing all models, so larger ones might timeout...")
- }
-
- chatModels := append(ollamaEngineChatModels, llamaRunnerChatModels...)
- chatModels = append(chatModels, mlxEngineChatModels...)
-
- for _, model := range testModels(chatModels) {
- t.Run(model, func(t *testing.T) {
- if time.Now().Sub(started) > softTimeout {
- t.Skip("skipping remaining tests to avoid excessive runtime")
- }
- pullOrSkip(ctx, t, client, model)
- if maxVram > 0 {
- resp, err := client.List(ctx)
- if err != nil {
- t.Fatalf("list models failed %v", err)
- }
- for _, m := range resp.Models {
- if m.Name == model && float32(m.Size)*1.2 > float32(maxVram) {
- t.Skipf("model %s is too large for available VRAM: %s > %s", model, format.HumanBytes(m.Size), format.HumanBytes(int64(maxVram)))
- }
- }
- }
- initialTimeout := 120 * time.Second
- streamTimeout := 30 * time.Second
- slog.Info("loading", "model", model)
- err := client.Generate(ctx,
- &api.GenerateRequest{Model: model, KeepAlive: &api.Duration{Duration: 10 * time.Second}},
- func(response api.GenerateResponse) error { return nil },
- )
- if err != nil {
- skipIfMLXUnsupported(t, err)
- t.Fatalf("failed to load model %s: %s", model, err)
- }
- gpuPercent := getGPUPercent(ctx, t, client, model)
- if gpuPercent < 80 {
- slog.Warn("Low GPU percentage - increasing timeouts", "percent", gpuPercent)
- initialTimeout = 240 * time.Second
- streamTimeout = 40 * time.Second
- }
-
- // TODO - fiddle with context size
- req := api.ChatRequest{
- Model: model,
- Messages: []api.Message{
- {
- Role: "user",
- Content: blueSkyPrompt,
- },
- },
- KeepAlive: &api.Duration{Duration: 10 * time.Second},
- Options: map[string]interface{}{
- "temperature": 0,
- "seed": 123,
- },
- }
- DoChat(ctx, t, client, req, blueSkyExpected, initialTimeout, streamTimeout)
- // best effort unload once we're done with the model
- client.Generate(ctx, &api.GenerateRequest{Model: req.Model, KeepAlive: &api.Duration{Duration: 0}}, func(rsp api.GenerateResponse) error { return nil })
- })
- }
-}
-
-func TestModelsEmbed(t *testing.T) {
- softTimeout, hardTimeout := getTimeouts(t)
- ctx, cancel := context.WithTimeout(context.Background(), hardTimeout)
- defer cancel()
- client, _, cleanup := InitServerConnection(ctx, t)
- defer cleanup()
-
- // TODO use info API eventually
- var maxVram uint64
- var err error
- if s := os.Getenv("OLLAMA_MAX_VRAM"); s != "" {
- maxVram, err = strconv.ParseUint(s, 10, 64)
- if err != nil {
- t.Fatalf("invalid OLLAMA_MAX_VRAM %v", err)
- }
- } else {
- slog.Warn("No VRAM info available, testing all models, so larger ones might timeout...")
- }
-
- data, err := ioutil.ReadFile(filepath.Join("testdata", "embed.json"))
- if err != nil {
- t.Fatalf("failed to open test data file: %s", err)
- }
- testCase := map[string][]float64{}
- err = json.Unmarshal(data, &testCase)
- if err != nil {
- t.Fatalf("failed to load test data: %s", err)
- }
- for model, expected := range testCase {
- if testModel != "" && model != testModel {
- continue
- }
-
- t.Run(model, func(t *testing.T) {
- if time.Now().Sub(started) > softTimeout {
- t.Skip("skipping remaining tests to avoid excessive runtime")
- }
- pullOrSkip(ctx, t, client, model)
- if maxVram > 0 {
- resp, err := client.List(ctx)
- if err != nil {
- t.Fatalf("list models failed %v", err)
- }
- for _, m := range resp.Models {
- if m.Name == model && float32(m.Size)*1.2 > float32(maxVram) {
- t.Skipf("model %s is too large for available VRAM: %s > %s", model, format.HumanBytes(m.Size), format.HumanBytes(int64(maxVram)))
- }
- }
- }
- req := api.EmbeddingRequest{
- Model: model,
- Prompt: "why is the sky blue?",
- KeepAlive: &api.Duration{Duration: 10 * time.Second},
- Options: map[string]interface{}{
- "temperature": 0,
- "seed": 123,
- },
- }
- resp, err := client.Embeddings(ctx, &req)
- if err != nil {
- t.Fatalf("embeddings call failed %s", err)
- }
- defer func() {
- // best effort unload once we're done with the model
- client.Generate(ctx, &api.GenerateRequest{Model: req.Model, KeepAlive: &api.Duration{Duration: 0}}, func(rsp api.GenerateResponse) error { return nil })
- }()
- if len(resp.Embedding) == 0 {
- t.Errorf("zero length embedding response")
- }
- if len(expected) != len(resp.Embedding) {
- expStr := make([]string, len(resp.Embedding))
- for i, v := range resp.Embedding {
- expStr[i] = fmt.Sprintf("%0.6f", v)
- }
- // When adding new models, use this output to populate the testdata/embed.json
- fmt.Printf("expected\n%s\n", strings.Join(expStr, ", "))
- t.Fatalf("expected %d, got %d", len(expected), len(resp.Embedding))
- }
- sim := cosineSimilarity(resp.Embedding, expected)
- if sim < 0.99 {
- t.Fatalf("expected %v, got %v (similarity: %f)", expected[0:5], resp.Embedding[0:5], sim)
- }
- })
- }
-
-}
diff --git a/integration/model_perf_test.go b/integration/model_perf_test.go
deleted file mode 100644
index d8e8c4bf4..000000000
--- a/integration/model_perf_test.go
+++ /dev/null
@@ -1,278 +0,0 @@
-//go:build integration && perf
-
-package integration
-
-import (
- "context"
- "fmt"
- "io/ioutil"
- "log/slog"
- "math"
- "os"
- "path/filepath"
- "strconv"
- "strings"
- "testing"
- "time"
-
- "github.com/ollama/ollama/api"
- "github.com/ollama/ollama/format"
-)
-
-var (
- // Models that don't work reliably with the large context prompt in this test case
- longContextFlakes = []string{
- "granite-code:latest",
- "nemotron-mini:latest",
- "falcon:latest", // 2k model
- "falcon2:latest", // 2k model
- "minicpm-v:latest",
- "qwen:latest",
- }
-)
-
-// Note: this test case can take a long time to run, particularly on models with
-// large contexts. Run with -timeout set to a large value to get reasonable coverage
-// Example usage:
-//
-// go test --tags=integration,perf -count 1 ./integration -v -timeout 90m -run TestModelsPerf 2>&1 | tee int.log
-// cat int.log | grep MODEL_PERF_HEADER | head -1| cut -f2- -d: > perf.csv
-// cat int.log | grep MODEL_PERF_DATA | cut -f2- -d: >> perf.csv
-func TestModelsPerf(t *testing.T) {
- doModelPerfTest(t, append(ollamaEngineChatModels, llamaRunnerChatModels...))
-}
-
-func TestLibraryModelsPerf(t *testing.T) {
- doModelPerfTest(t, libraryChatModels)
-}
-
-func doModelPerfTest(t *testing.T, chatModels []string) {
- softTimeout, hardTimeout := getTimeouts(t)
- slog.Info("Setting timeouts", "soft", softTimeout, "hard", hardTimeout)
- ctx, cancel := context.WithTimeout(context.Background(), hardTimeout)
- defer cancel()
- client, _, cleanup := InitServerConnection(ctx, t)
- defer cleanup()
-
- // TODO use info API eventually
- var maxVram uint64
- var err error
- if s := os.Getenv("OLLAMA_MAX_VRAM"); s != "" {
- maxVram, err = strconv.ParseUint(s, 10, 64)
- if err != nil {
- t.Fatalf("invalid OLLAMA_MAX_VRAM %v", err)
- }
- } else {
- slog.Warn("No VRAM info available, testing all models, so larger ones might timeout...")
- }
-
- data, err := ioutil.ReadFile(filepath.Join("testdata", "shakespeare.txt"))
- if err != nil {
- t.Fatalf("failed to open test data file: %s", err)
- }
- longPrompt := "summarize the following: " + string(data)
-
- targetArch := os.Getenv("OLLAMA_TEST_ARCHITECTURE")
-
- for _, model := range chatModels {
- if !strings.Contains(model, ":") {
- model = model + ":latest"
- }
- t.Run(model, func(t *testing.T) {
- if time.Now().Sub(started) > softTimeout {
- t.Skip("skipping remaining tests to avoid excessive runtime")
- }
- pullOrSkip(ctx, t, client, model)
- var maxContext int
-
- resp, err := client.Show(ctx, &api.ShowRequest{Model: model})
- if err != nil {
- t.Fatalf("show failed: %s", err)
- }
- arch := resp.ModelInfo["general.architecture"].(string)
- maxContext = int(resp.ModelInfo[fmt.Sprintf("%s.context_length", arch)].(float64))
- if targetArch != "" && arch != targetArch {
- t.Skip(fmt.Sprintf("Skipping %s architecture %s != %s", model, arch, targetArch))
- }
-
- if maxVram > 0 {
- resp, err := client.List(ctx)
- if err != nil {
- t.Fatalf("list models failed %v", err)
- }
- for _, m := range resp.Models {
- // For these tests we want to exercise a some amount of overflow on the CPU
- if m.Name == model && float32(m.Size)*0.75 > float32(maxVram) {
- t.Skipf("model %s is too large %s for available VRAM %s", model, format.HumanBytes(m.Size), format.HumanBytes(int64(maxVram)))
- }
- }
- }
- slog.Info("scneario", "model", model, "max_context", maxContext)
- loaded := false
- defer func() {
- // best effort unload once we're done with the model
- if loaded {
- client.Generate(ctx, &api.GenerateRequest{Model: model, KeepAlive: &api.Duration{Duration: 0}}, func(rsp api.GenerateResponse) error { return nil })
- }
- }()
-
- // Some models don't handle the long context data well so skip them to avoid flaky test results
- longContextFlake := false
- for _, flake := range longContextFlakes {
- if model == flake {
- longContextFlake = true
- break
- }
- }
-
- // iterate through a few context sizes for coverage without excessive runtime
- var contexts []int
- keepGoing := true
- if maxContext > 16384 {
- contexts = []int{4096, 8192, 16384, maxContext}
- } else if maxContext > 8192 {
- contexts = []int{4096, 8192, maxContext}
- } else if maxContext > 4096 {
- contexts = []int{4096, maxContext}
- } else if maxContext > 0 {
- contexts = []int{maxContext}
- } else {
- t.Fatal("unknown max context size")
- }
- for _, numCtx := range contexts {
- if !keepGoing && numCtx > 8192 { // Always try up to 8k before bailing out
- break
- }
- skipLongPrompt := false
-
- // Workaround bug 11172 temporarily...
- maxPrompt := longPrompt
- // If we fill the context too full with the prompt, many models
- // quickly hit context shifting and go bad.
- if len(maxPrompt) > numCtx*2 { // typically yields ~1/2 full context
- maxPrompt = maxPrompt[:numCtx*2]
- }
-
- testCases := []struct {
- prompt string
- anyResp []string
- }{
- {blueSkyPrompt, blueSkyExpected},
- {maxPrompt, []string{"shakespeare", "oppression", "sorrows", "gutenberg", "child", "license", "sonnet", "melancholy", "love", "sorrow", "beauty"}},
- }
- var gpuPercent int
- for _, tc := range testCases {
- if len(tc.prompt) > 100 && (longContextFlake || skipLongPrompt) {
- slog.Info("skipping long prompt", "model", model, "num_ctx", numCtx, "gpu_percent", gpuPercent)
- continue
- }
- req := api.ChatRequest{
- Model: model,
- Messages: []api.Message{
- {
- Role: "user",
- Content: tc.prompt,
- },
- },
- KeepAlive: &api.Duration{Duration: 20 * time.Second}, // long enough to ensure a ps returns
- Options: map[string]interface{}{
- "temperature": 0,
- "seed": 123,
- "num_ctx": numCtx,
- },
- }
- atLeastOne := false
- var resp api.ChatResponse
-
- stream := false
- req.Stream = &stream
-
- // Avoid potentially getting stuck indefinitely
- limit := 5 * time.Minute
- genCtx, cancel := context.WithDeadlineCause(
- ctx,
- time.Now().Add(limit),
- fmt.Errorf("generate on model %s with ctx %d took longer than %v", model, numCtx, limit),
- )
- defer cancel()
-
- err = client.Chat(genCtx, &req, func(rsp api.ChatResponse) error {
- resp = rsp
- return nil
- })
- if err != nil {
- // Avoid excessive test runs, but don't consider a failure with massive context
- if numCtx > 16384 && strings.Contains(err.Error(), "took longer") {
- slog.Warn("max context was taking too long, skipping", "error", err)
- keepGoing = false
- skipLongPrompt = true
- continue
- }
- t.Fatalf("generate error: ctx:%d err:%s", numCtx, err)
- }
- loaded = true
- for _, expResp := range tc.anyResp {
- if strings.Contains(strings.ToLower(resp.Message.Content), expResp) {
- atLeastOne = true
- break
- }
- }
- if !atLeastOne {
- t.Fatalf("response didn't contain expected values: ctx:%d expected:%v response:%s ", numCtx, tc.anyResp, resp.Message.Content)
- }
- models, err := client.ListRunning(ctx)
- if err != nil {
- slog.Warn("failed to list running models", "error", err)
- continue
- }
- if len(models.Models) > 1 {
- slog.Warn("multiple models loaded, may impact performance results", "loaded", models.Models)
- }
- for _, m := range models.Models {
- if m.Name == model {
- if m.SizeVRAM == 0 {
- slog.Info("Model fully loaded into CPU")
- gpuPercent = 0
- keepGoing = false
- skipLongPrompt = true
- } else if m.SizeVRAM == m.Size {
- slog.Info("Model fully loaded into GPU")
- gpuPercent = 100
- } else {
- sizeCPU := m.Size - m.SizeVRAM
- cpuPercent := math.Round(float64(sizeCPU) / float64(m.Size) * 100)
- gpuPercent = int(100 - cpuPercent)
- slog.Info("Model split between CPU/GPU", "CPU", cpuPercent, "GPU", gpuPercent)
- keepGoing = false
-
- // Heuristic to avoid excessive test run time
- if gpuPercent < 90 {
- skipLongPrompt = true
- }
- }
- }
- }
- // Round the logged prompt count for comparisons across versions/configurations which can vary slightly
- fmt.Fprintf(os.Stderr, "MODEL_PERF_HEADER:%s,%s,%s,%s,%s,%s,%s\n",
- "MODEL",
- "CONTEXT",
- "GPU PERCENT",
- "APPROX PROMPT COUNT",
- "LOAD TIME",
- "PROMPT EVAL TPS",
- "EVAL TPS",
- )
- fmt.Fprintf(os.Stderr, "MODEL_PERF_DATA:%s,%d,%d,%d,%0.2f,%0.2f,%0.2f\n",
- model,
- numCtx,
- gpuPercent,
- (resp.PromptEvalCount/10)*10,
- float64(resp.LoadDuration)/1000000000.0,
- float64(resp.PromptEvalCount)/(float64(resp.PromptEvalDuration)/1000000000.0),
- float64(resp.EvalCount)/(float64(resp.EvalDuration)/1000000000.0),
- )
- }
- }
- })
- }
-}
diff --git a/integration/quantization_test.go b/integration/quantization_test.go
index 9de401a47..f55e4cd60 100644
--- a/integration/quantization_test.go
+++ b/integration/quantization_test.go
@@ -1,4 +1,4 @@
-//go:build integration && models
+//go:build integration && release
package integration
@@ -14,7 +14,11 @@ import (
"github.com/ollama/ollama/api"
)
-func TestQuantization(t *testing.T) {
+func runQuantization(t *testing.T) {
+ if testModel != "" {
+ t.Skip("exercises quantization with a fixed source model, not applicable with model override")
+ }
+
sourceModels := []string{
"qwen2.5:0.5b-instruct-fp16",
}
diff --git a/integration/reg_fast_test.go b/integration/reg_fast_test.go
new file mode 100644
index 000000000..60486cfcb
--- /dev/null
+++ b/integration/reg_fast_test.go
@@ -0,0 +1,48 @@
+//go:build integration && fast
+
+package integration
+
+var (
+ fastNumPredictModel = "llama3.2:1b"
+ fastChatModels = []integrationModel{
+ {Name: "gemma4", MinVRAMGB: 8},
+ {Name: "gemma4:12b", MinVRAMGB: 16},
+ {Name: "qwen3.5:2b-nvfp4", MinVRAMGB: 4},
+ }
+ fastEmbedModels = []string{"qwen3-embedding"}
+ fastVisionTextModels = []string{"gemma4"}
+ fastToolsModels = []string{"qwen3.5:2b"}
+ fastToolsStressModels = []string{"lfm2.5"}
+ fastAudioModels = []string{"gemma4:e2b"}
+)
+
+func init() {
+ // API/basic/context/concurrency smoke cases
+ registerIntegrationCases(
+ integrationTestCase("api-generate", smol, runAPIGenerate),
+ integrationTestCase("api-chat", smol, runAPIChat),
+ integrationTestCase("api-list-models", "", runAPIListModels),
+ integrationTestCase("api-show-model", "llama3.2", runAPIShowModel),
+ integrationTestCase("generate-logprobs", smol, runAPIGenerateLogprobs),
+ integrationTestCase("chat-logprobs", smol, runAPIChatLogprobs),
+ integrationTestCase("blue-sky", smol, runBlueSky),
+ integrationTestCase("thinking-enabled", smol, runThinkingEnabled),
+ integrationTestCase("thinking-suppressed", smol, runThinkingSuppressed),
+ integrationModelTestCase("num-predict", fastNumPredictModel, runNumPredict),
+ integrationTestCase("embedding-api", "all-minilm", runAllMiniLMEmbeddings),
+ integrationTestCase("embed-api-truncate", "all-minilm", runAllMiniLMEmbedTruncate),
+ integrationTestCase("context-long-input", smol, runLongInputContext),
+ integrationTestCase("context-exhaustion", smol, runContextExhaustion),
+ integrationTestCase("generate-history", smol, runGenerateWithHistory),
+ integrationTestCase("concurrent-chat", smol, runConcurrentChat),
+ )
+
+ // Model-parametric cases
+ registerModelMinVRAM(fastChatModels)
+ registerChatCases(testModels(modelNames(fastChatModels)))
+ registerEmbeddingCases(testModels(fastEmbedModels))
+ registerVisionTextCases(testModels(fastVisionTextModels))
+ registerToolCases(testModels(fastToolsModels))
+ registerToolStressCases(testModels(fastToolsStressModels))
+ registerAudioTranscriptionCases(testModels(fastAudioModels))
+}
diff --git a/integration/reg_groups_test.go b/integration/reg_groups_test.go
new file mode 100644
index 000000000..ed5a9d8ff
--- /dev/null
+++ b/integration/reg_groups_test.go
@@ -0,0 +1,143 @@
+//go:build integration && (fast || release || library)
+
+package integration
+
+import (
+ "strings"
+ "testing"
+)
+
+func runIntegrationGroup(t *testing.T, cases ...string) {
+ t.Helper()
+ selected := map[string]struct{}{}
+ for _, c := range cases {
+ selected[c] = struct{}{}
+ }
+ var ran bool
+ for _, c := range integrationCases {
+ if _, ok := selected[c.Case]; !ok {
+ continue
+ }
+ ran = true
+ c := c
+ name := c.Case
+ if c.Model != "" {
+ name += "/" + testName(c.Model)
+ }
+ t.Run(name, c.Run)
+ }
+ if !ran {
+ t.Skip("no integration cases selected")
+ }
+}
+
+func TestAPI(t *testing.T) {
+ runIntegrationGroup(t,
+ "api-generate",
+ "api-chat",
+ "api-list-models",
+ "api-show-model",
+ "generate-logprobs",
+ "chat-logprobs",
+ )
+}
+
+func TestBasic(t *testing.T) {
+ runIntegrationGroup(t,
+ "blue-sky",
+ "unicode-input",
+ "unicode-output",
+ "unicode-model-dir",
+ "num-predict",
+ "thinking-enabled",
+ "thinking-suppressed",
+ )
+}
+
+func TestChat(t *testing.T) {
+ runIntegrationGroup(t,
+ "chat",
+ "chat-history",
+ )
+}
+
+func TestEmbedding(t *testing.T) {
+ runIntegrationGroup(t,
+ "embed",
+ "embed-correlation",
+ "embedding-api",
+ "embed-api",
+ "embed-api-batch",
+ "embed-api-truncate",
+ "embed-truncation",
+ "embed-large-input",
+ "embed-status-code",
+ )
+}
+
+func TestVision(t *testing.T) {
+ runIntegrationGroup(t,
+ "vision-multiturn",
+ "vision-count",
+ "vision-scene",
+ "vision-spatial",
+ "vision-detail",
+ "vision-multi-image",
+ "vision-description",
+ "vision-split-batch",
+ "vision-text",
+ )
+}
+
+func TestAudio(t *testing.T) {
+ runIntegrationGroup(t,
+ "audio-transcription",
+ "audio-response",
+ "openai-audio-transcription",
+ "openai-chat-audio",
+ )
+}
+
+func TestContext(t *testing.T) {
+ runIntegrationGroup(t,
+ "context-long-input",
+ "context-exhaustion",
+ "generate-history",
+ "parallel-generate-history",
+ "parallel-chat-history",
+ )
+}
+
+func TestConcurrency(t *testing.T) {
+ runIntegrationGroup(t,
+ "concurrent-chat",
+ "scheduler-multimodel",
+ "scheduler-max-queue",
+ )
+}
+
+func TestTools(t *testing.T) {
+ runIntegrationGroup(t,
+ "tools",
+ "tools-stress",
+ )
+}
+
+func TestCreate(t *testing.T) {
+ runIntegrationGroup(t,
+ "create-safetensors",
+ "create-gguf",
+ )
+}
+
+func TestQuantization(t *testing.T) {
+ runIntegrationGroup(t, "quantization")
+}
+
+func TestImageGeneration(t *testing.T) {
+ runIntegrationGroup(t, "image-generation")
+}
+
+func testName(s string) string {
+ return strings.NewReplacer("/", "~", " ", "_").Replace(s)
+}
diff --git a/integration/reg_library_test.go b/integration/reg_library_test.go
new file mode 100644
index 000000000..1da3cc5d0
--- /dev/null
+++ b/integration/reg_library_test.go
@@ -0,0 +1,231 @@
+//go:build integration && library
+
+package integration
+
+// Broad public library inventory, roughly newest to oldest from
+// https://ollama.com/library?sort=newest. Cloud-only models are omitted;
+// models with only very large local tags are kept commented in place.
+var libraryModels = []string{
+ "lfm2.5",
+ "mistral-medium-3.5",
+ "granite4.1",
+ "nemotron3",
+ "laguna-xs.2",
+ "qwen3.6",
+ "medgemma1.5",
+ "medgemma",
+ "nemotron-cascade-2",
+ "gemma4",
+ "lfm2",
+ "nemotron-3-super",
+ "qwen3.5",
+ "qwen3-coder-next",
+ "glm-ocr",
+ "lfm2.5-thinking",
+ "glm-4.7-flash",
+ "translategemma",
+ "nemotron-3-nano",
+ "functiongemma",
+ "olmo-3.1",
+ "olmo-3",
+ "nomic-embed-text-v2-moe",
+ "devstral-small-2",
+ "rnj-1",
+ "devstral-2",
+ "qwen3-next",
+ "ministral-3",
+ "deepseek-ocr",
+ // "cogito-2.1", // only local tags are very large (404.0GB minimum)
+ "gpt-oss-safeguard",
+ "qwen3-vl",
+ "granite4",
+ "qwen3-embedding",
+ "embeddinggemma",
+ // "deepseek-v3.1", // only local tags are very large (404.0GB minimum)
+ "gpt-oss",
+ "qwen3-coder",
+ "mistral-small3.2",
+ "gemma3n",
+ "magistral",
+ "devstral",
+ "qwen2.5vl",
+ "phi4-reasoning",
+ "phi4-mini-reasoning",
+ "qwen3",
+ "granite3.3",
+ "deepcoder",
+ "mistral-small3.1",
+ "cogito",
+ "llama4",
+ "exaone-deep",
+ "command-a",
+ "gemma3",
+ "command-r7b-arabic",
+ "granite3.2-vision",
+ "phi4-mini",
+ "granite3.2",
+ "r1-1776",
+ "deepscaler",
+ "openthinker",
+ "deepseek-r1",
+ "olmo2",
+ "command-r7b",
+ // "deepseek-v3", // only local tags are very large (404.0GB minimum)
+ "phi4",
+ "dolphin3",
+ "smallthinker",
+ "granite3.1-dense",
+ "granite3.1-moe",
+ "falcon3",
+ "granite-embedding",
+ "exaone3.5",
+ "llama3.3",
+ "snowflake-arctic-embed2",
+ "sailor2",
+ "qwq",
+ "marco-o1",
+ "tulu3",
+ "athene-v2",
+ "opencoder",
+ "llama3.2-vision",
+ "smollm2",
+ "granite3-guardian",
+ "aya-expanse",
+ "granite3-dense",
+ "granite3-moe",
+ "nemotron",
+ "shieldgemma",
+ "llama-guard3",
+ "llama3.2",
+ "qwen2.5-coder",
+ "solar-pro",
+ "nemotron-mini",
+ "qwen2.5",
+ "bespoke-minicheck",
+ "mistral-small",
+ "reader-lm",
+ "minicpm-v",
+ // "deepseek-v2.5", // only local tags are very large (133.0GB minimum)
+ "reflection",
+ "yi-coder",
+ "qwen2-math",
+ "hermes3",
+ "phi3.5",
+ "smollm",
+ "bge-large",
+ "paraphrase-multilingual",
+ "bge-m3",
+ "mistral-large",
+ "llama3.1",
+ "nuextract",
+ "mistral-nemo",
+ "firefunction-v2",
+ "llama3-groq-tool-use",
+ "mathstral",
+ "codegeex4",
+ "glm4",
+ "internlm2",
+ "gemma2",
+ "deepseek-coder-v2",
+ "qwen2",
+ "deepseek-v2",
+ "codestral",
+ "granite-code",
+ "aya",
+ "falcon2",
+ "llama3-chatqa",
+ "llava-phi3",
+ "llava-llama3",
+ "llama3-gradient",
+ "moondream",
+ "phi3",
+ "dolphin-llama3",
+ "llama3",
+ "codeqwen",
+ "snowflake-arctic-embed",
+ "dbrx",
+ "command-r-plus",
+ "wizardlm2",
+ "codegemma",
+ "command-r",
+ "mxbai-embed-large",
+ "dolphincoder",
+ "starcoder2",
+ "all-minilm",
+ "nomic-embed-text",
+ "gemma",
+ "stablelm2",
+ "duckdb-nsql",
+ "qwen",
+ "tinydolphin",
+ "stable-code",
+ "nous-hermes2-mixtral",
+ "megadolphin",
+ "llama-pro",
+ "tinyllama",
+ "openhermes",
+ "notux",
+ "notus",
+ "dolphin-mistral",
+ "nous-hermes2",
+ "dolphin-phi",
+ "phi",
+ "solar",
+ "dolphin-mixtral",
+ "mixtral",
+ "bakllava",
+ "llava",
+ "stablelm-zephyr",
+ "magicoder",
+ "deepseek-llm",
+ "meditron",
+ "starling-lm",
+ "orca2",
+ "deepseek-coder",
+ "alfred",
+ "goliath",
+ "neural-chat",
+ "openchat",
+ "yi",
+ "yarn-mistral",
+ "yarn-llama2",
+ "xwinlm",
+ "mistrallite",
+ "codebooga",
+ "mistral-openorca",
+ "zephyr",
+ "nexusraven",
+ "samantha-mistral",
+ "starcoder",
+ "sqlcoder",
+ "mistral",
+ "falcon",
+ "wizardcoder",
+ "phind-codellama",
+ "codellama",
+ "wizardlm",
+ "wizardlm-uncensored",
+ "wizard-vicuna",
+ "wizard-vicuna-uncensored",
+ "wizard-math",
+ "vicuna",
+ "stable-beluga",
+ "orca-mini",
+ "open-orca-platypus2",
+ "nous-hermes",
+ "medllama2",
+ "llama2",
+ "llama2-uncensored",
+ "llama2-chinese",
+ "everythinglm",
+ "codeup",
+}
+
+func init() {
+ // Broad model sweeps. Each case skips models that do not expose the
+ // capability it is testing.
+ registerChatCases(testModels(libraryModels))
+ registerLibraryEmbeddingCases(testModels(libraryModels))
+ registerToolCases(testModels(libraryModels))
+ registerVisionTextCases(testModels(libraryModels))
+}
diff --git a/integration/reg_release_test.go b/integration/reg_release_test.go
new file mode 100644
index 000000000..5f2b2e46e
--- /dev/null
+++ b/integration/reg_release_test.go
@@ -0,0 +1,132 @@
+//go:build integration && release
+
+package integration
+
+var (
+ releaseUnicodeInputModel = integrationModel{Name: "deepseek-coder-v2:16b-lite-instruct-q2_K", MinVRAMGB: 12}
+ releaseUnicodeOutputModel = "gemma2:2b"
+ releaseNumPredictModel = "llama3.2:1b"
+ releaseParallelHistoryModel = integrationModel{Name: "gpt-oss:20b", MinVRAMGB: 16}
+ releaseChatModels = []integrationModel{
+ {Name: "gemma4", MinVRAMGB: 8},
+ {Name: "gemma4:12b", MinVRAMGB: 16},
+ {Name: "lfm2.5", MinVRAMGB: 6},
+ {Name: "granite4.1:8b", MinVRAMGB: 6},
+ {Name: "gpt-oss:20b", MinVRAMGB: 16},
+ {Name: "qwen3.6:27b", MinVRAMGB: 20},
+ {Name: "qwen3.5:2b", MinVRAMGB: 4},
+ {Name: "qwen3.5:2b-nvfp4", MinVRAMGB: 4},
+ {Name: "deepseek-r1:8b", MinVRAMGB: 6},
+ {Name: "mistral-small3.2:latest", MinVRAMGB: 16},
+ {Name: "llama3.2:latest"},
+ {Name: "gemma4:e2b-nvfp4", MinVRAMGB: 8},
+ }
+ releaseEmbedModels = []string{
+ "embeddinggemma",
+ "nomic-embed-text",
+ "all-minilm",
+ "bge-large",
+ "bge-m3",
+ "granite-embedding",
+ "mxbai-embed-large",
+ "paraphrase-multilingual",
+ "snowflake-arctic-embed",
+ "snowflake-arctic-embed2",
+ "qwen3-embedding",
+ }
+ releaseVisionModels = []string{
+ "nemotron3:33b",
+ "gemma4",
+ "qwen3.6:27b",
+ // "llama3.2-vision", // TODO: re-enable when llama.cpp supports mllama.
+ }
+ releaseVisionTextModels = []string{
+ "gemma4",
+ "qwen3.6:27b",
+ "qwen3.5:2b",
+ // "llama3.2-vision", // TODO: re-enable when llama.cpp supports mllama.
+ "ministral-3:3b",
+ }
+ releaseToolsModels = []string{
+ "lfm2.5",
+ "nemotron3:33b",
+ "gemma4",
+ "gpt-oss:20b",
+ "qwen3.6:27b",
+ }
+ releaseAudioModels = []string{
+ "nemotron3:33b",
+ "gemma4:e2b",
+ "gemma4:e4b",
+ }
+)
+
+const releaseSplitBatchVisionModel = "qwen3.5:2b"
+
+func init() {
+ // Fixed release regression cases
+ registerIntegrationCases(
+ integrationTestCase("api-generate", smol, runAPIGenerate),
+ integrationTestCase("api-chat", smol, runAPIChat),
+ integrationTestCase("api-list-models", "", runAPIListModels),
+ integrationTestCase("api-show-model", "llama3.2", runAPIShowModel),
+ integrationTestCase("generate-logprobs", smol, runAPIGenerateLogprobs),
+ integrationTestCase("chat-logprobs", smol, runAPIChatLogprobs),
+
+ integrationTestCase("blue-sky", smol, runBlueSky),
+ integrationModelTestCase("unicode-input", releaseUnicodeInputModel.Name, runUnicode),
+ integrationModelTestCase("unicode-output", releaseUnicodeOutputModel, runExtendedUnicodeOutput),
+ integrationTestCase("unicode-model-dir", smol, runUnicodeModelDir),
+ integrationModelTestCase("num-predict", releaseNumPredictModel, runNumPredict),
+
+ integrationModelsTestCase("embed-correlation", releaseEmbedModels, runEmbedCosineDistanceCorrelation),
+ integrationTestCase("embedding-api", "all-minilm", runAllMiniLMEmbeddings),
+ integrationTestCase("embed-api", "all-minilm", runAllMiniLMEmbed),
+ integrationTestCase("embed-api-batch", "all-minilm", runAllMiniLMBatchEmbed),
+ integrationTestCase("embed-api-truncate", "all-minilm", runAllMiniLMEmbedTruncate),
+ integrationModelsTestCase("embed-truncation", releaseEmbedModels, runEmbedTruncation),
+ integrationModelsTestCase("embed-large-input", releaseEmbedModels, runEmbedLargeInput),
+ integrationModelsTestCase("embed-status-code", releaseEmbedModels, runEmbedStatusCode),
+
+ integrationModelsTestCase("vision-multiturn", releaseVisionModels, runVisionMultiTurn),
+ integrationModelsTestCase("vision-count", releaseVisionModels, runVisionObjectCounting),
+ integrationModelsTestCase("vision-scene", releaseVisionModels, runVisionSceneUnderstanding),
+ integrationModelsTestCase("vision-spatial", releaseVisionModels, runVisionSpatialReasoning),
+ integrationModelsTestCase("vision-detail", releaseVisionModels, runVisionDetailRecognition),
+ integrationModelsTestCase("vision-multi-image", releaseVisionModels, runVisionMultiImage),
+ integrationModelsTestCase("vision-description", releaseVisionModels, runVisionImageDescription),
+ integrationModelTestCase("vision-split-batch", releaseSplitBatchVisionModel, runIntegrationSplitBatch),
+
+ integrationModelsTestCase("audio-response", releaseAudioModels, runAudioResponse),
+ integrationModelsTestCase("openai-audio-transcription", releaseAudioModels, runOpenAIAudioTranscription),
+ integrationModelsTestCase("openai-chat-audio", releaseAudioModels, runOpenAIChatWithAudio),
+
+ integrationTestCase("context-long-input", smol, runLongInputContext),
+ integrationTestCase("context-exhaustion", smol, runContextExhaustion),
+ integrationModelTestCase("parallel-generate-history", releaseParallelHistoryModel.Name, runParallelGenerateWithHistory),
+ integrationTestCase("generate-history", smol, runGenerateWithHistory),
+ integrationModelTestCase("parallel-chat-history", defaultTestModel(releaseParallelHistoryModel.Name), runParallelChatWithHistory),
+ integrationTestCase("chat-history", smol, runChatWithHistory),
+ integrationTestCase("concurrent-chat", smol, runConcurrentChat),
+ integrationTestCase("scheduler-multimodel", "", runMultiModelStress),
+ integrationTestCase("scheduler-max-queue", smol, runMaxQueue),
+
+ integrationTestCase("thinking-enabled", smol, runThinkingEnabled),
+ integrationTestCase("thinking-suppressed", smol, runThinkingSuppressed),
+
+ integrationTestCase("create-safetensors", "", runCreateSafetensorsLLM),
+ integrationTestCase("create-gguf", "", runCreateGGUF),
+ integrationTestCase("quantization", "qwen2.5:0.5b-instruct-fp16", runQuantization),
+ integrationTestCase("image-generation", "", runImageGeneration),
+ )
+
+ // Model-parametric cases
+ registerModelMinVRAM([]integrationModel{releaseUnicodeInputModel, releaseParallelHistoryModel})
+ registerModelMinVRAM(releaseChatModels)
+ registerChatCases(testModels(modelNames(releaseChatModels)))
+ registerEmbeddingCases(testModels(releaseEmbedModels))
+ registerVisionTextCases(testModels(releaseVisionTextModels))
+ registerToolCases(testModels(releaseToolsModels))
+ registerToolStressCases(testModels(releaseToolsModels))
+ registerAudioTranscriptionCases(testModels(releaseAudioModels))
+}
diff --git a/integration/reg_required_test.go b/integration/reg_required_test.go
new file mode 100644
index 000000000..c178ce627
--- /dev/null
+++ b/integration/reg_required_test.go
@@ -0,0 +1,9 @@
+//go:build integration && !fast && !release && !library && !imagegen
+
+package integration
+
+import "testing"
+
+func TestIntegrationRequiresScope(t *testing.T) {
+ t.Fatal("integration tests require one of the fast, release, or library tags")
+}
diff --git a/integration/reg_test.go b/integration/reg_test.go
new file mode 100644
index 000000000..529b0ce22
--- /dev/null
+++ b/integration/reg_test.go
@@ -0,0 +1,140 @@
+//go:build integration
+
+package integration
+
+import "testing"
+
+type integrationCase struct {
+ Key string
+ Case string
+ Model string
+ Run func(t *testing.T)
+}
+
+type integrationModel struct {
+ Name string
+ MinVRAMGB uint64
+}
+
+var (
+ integrationCases []integrationCase
+ integrationCaseKeys = map[string]struct{}{}
+ modelMinVRAMGB = map[string]uint64{}
+)
+
+func registerIntegrationCases(cases ...integrationCase) {
+ for _, c := range cases {
+ if _, ok := integrationCaseKeys[c.Key]; ok {
+ continue
+ }
+ integrationCaseKeys[c.Key] = struct{}{}
+ integrationCases = append(integrationCases, c)
+ }
+}
+
+func integrationTestCase(name, model string, run func(t *testing.T)) integrationCase {
+ key := name
+ if model != "" {
+ key += "/" + model
+ }
+ return integrationCase{
+ Key: key,
+ Case: name,
+ Model: model,
+ Run: run,
+ }
+}
+
+func integrationModelTestCase(name, model string, run func(*testing.T, string)) integrationCase {
+ return integrationTestCase(name, model, func(t *testing.T) {
+ run(t, model)
+ })
+}
+
+func integrationModelsTestCase(name string, models []string, run func(*testing.T, []string)) integrationCase {
+ return integrationTestCase(name, "", func(t *testing.T) {
+ run(t, models)
+ })
+}
+
+func registerModelIntegrationCases(name string, models []string, run func(*testing.T, string)) {
+ cases := make([]integrationCase, 0, len(models))
+ for _, model := range models {
+ model := model
+ cases = append(cases, integrationCase{
+ Key: name + "/" + model,
+ Case: name,
+ Model: model,
+ Run: func(t *testing.T) {
+ run(t, model)
+ },
+ })
+ }
+ registerIntegrationCases(cases...)
+}
+
+func modelNames(models []integrationModel) []string {
+ names := make([]string, 0, len(models))
+ for _, model := range models {
+ names = append(names, model.Name)
+ }
+ return names
+}
+
+func registerModelMinVRAM(models []integrationModel) {
+ for _, model := range models {
+ if model.MinVRAMGB > 0 {
+ modelMinVRAMGB[model.Name] = model.MinVRAMGB
+ }
+ }
+}
+
+func skipRegisteredMinVRAM(t *testing.T, model string) {
+ t.Helper()
+ if v, ok := modelMinVRAMGB[model]; ok {
+ skipUnderMinVRAM(t, v)
+ }
+}
+
+type knownIntegrationFlake struct {
+ Scenario string
+ Model string
+ Reason string
+}
+
+var knownIntegrationFlakes = []knownIntegrationFlake{
+ {
+ Scenario: "tools-stress/multi_turn",
+ Model: "gemma4",
+ Reason: "returns an empty response on the agent-style multi-turn tool prompt",
+ },
+ {
+ Scenario: "tools-stress/multi_turn",
+ Model: "qwen3.5:2b",
+ Reason: "returns an empty response after the tool result in the agent-style multi-turn prompt",
+ },
+ {
+ Scenario: "vision-text",
+ Model: "qwen3.5:2b",
+ Reason: "times out instead of returning OCR text for the Ollamas image",
+ },
+ {
+ Scenario: "vision-multiturn",
+ Model: "gemma4",
+ Reason: "counts five animals in the Ollamas image instead of four",
+ },
+ {
+ Scenario: "vision-count",
+ Model: "gemma4",
+ Reason: "counts five animals in the docs image instead of four",
+ },
+}
+
+func skipKnownIntegrationFlake(t *testing.T, scenario, model string) {
+ t.Helper()
+ for _, flake := range knownIntegrationFlakes {
+ if flake.Scenario == scenario && flake.Model == model {
+ t.Skipf("known model/scenario flake: %s", flake.Reason)
+ }
+ }
+}
diff --git a/integration/thinking_test.go b/integration/thinking_test.go
index 15c8b0740..4a94178bf 100644
--- a/integration/thinking_test.go
+++ b/integration/thinking_test.go
@@ -11,9 +11,9 @@ import (
"github.com/ollama/ollama/api"
)
-// TestThinkingEnabled verifies that when thinking is requested, the model
+// runThinkingEnabled verifies that when thinking is requested, the model
// produces both thinking and content output without leaking raw channel tags.
-func TestThinkingEnabled(t *testing.T) {
+func runThinkingEnabled(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
@@ -84,9 +84,9 @@ func TestThinkingEnabled(t *testing.T) {
}
}
-// TestThinkingSuppressed verifies that when thinking is NOT requested,
+// runThinkingSuppressed verifies that when thinking is NOT requested,
// the model does not leak thinking/channel content into the response.
-func TestThinkingSuppressed(t *testing.T) {
+func runThinkingSuppressed(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
diff --git a/integration/tools_stress_test.go b/integration/tools_stress_test.go
index a849e5575..f42b010bd 100644
--- a/integration/tools_stress_test.go
+++ b/integration/tools_stress_test.go
@@ -15,117 +15,90 @@ import (
"github.com/ollama/ollama/api"
)
-// TestAPIToolCallingStress tests tool calling with complex, agent-style prompts
-// that include large system messages, multiple tools, and multi-turn conversations.
-// This catches cache corruption and parser bugs that simple tool tests miss.
-func TestAPIToolCallingStress(t *testing.T) {
+func registerToolStressCases(models []string) {
+ registerModelIntegrationCases("tools-stress", models, runAPIToolCallingStressModel)
+}
+
+var toolStressSkipModels = map[string]string{
+ "lfm2.5-thinking": "returns text instead of tool calls with complex system prompts",
+ "qwen3.5:2b": "2B model too small for reliable multi-tool agent prompts",
+ "qwen3-vl": "vision model, extremely slow with complex tool prompts",
+ "llama3.2": "3B model too small for reliable multi-tool agent prompts",
+ "mistral": "7B v0.3 returns text instead of tool calls with complex prompts",
+ "mixtral:8x22b": "returns text instead of tool calls with complex prompts",
+ "qwen2": "returns text instead of tool calls with complex prompts",
+ "granite3.3": "returns text instead of tool calls with complex prompts",
+}
+
+func runAPIToolCallingStressModel(t *testing.T, model string) {
initialTimeout := 120 * time.Second
streamTimeout := 120 * time.Second
+ softTimeout, _ := getTimeouts(t)
+ if time.Since(started) > softTimeout {
+ t.Skip("skipping remaining tests to avoid excessive runtime")
+ }
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
- minVRAM := map[string]uint64{
- "qwen3-vl": 16,
- "gpt-oss:20b": 16,
- "gpt-oss:120b": 70,
- "qwen3": 6,
- "llama3.1": 8,
- "llama3.2": 4,
- "mistral": 6,
- "qwen2.5": 6,
- "qwen2": 6,
- "ministral-3": 20,
- "mistral-nemo": 9,
- "mistral-small": 16,
- "mixtral:8x22b": 80,
- "qwq": 20,
- "granite3.3": 7,
- }
-
- // Models that don't reliably produce tool calls with complex/multi-tool prompts.
- // The stress test uses a large system prompt with many tools, simulating coding agents.
- // Some models are too small, too slow, or not designed for this use case.
- skipModels := map[string]string{
- "lfm2.5-thinking": "returns text instead of tool calls with complex system prompts",
- "qwen3-vl": "vision model, extremely slow with complex tool prompts",
- "llama3.2": "3B model too small for reliable multi-tool agent prompts",
- "mistral": "7B v0.3 returns text instead of tool calls with complex prompts",
- "mixtral:8x22b": "returns text instead of tool calls with complex prompts",
- "qwen2": "returns text instead of tool calls with complex prompts",
- "granite3.3": "returns text instead of tool calls with complex prompts",
- }
-
- models := testModels(libraryToolsModels)
-
- softTimeout, _ := getTimeouts(t)
-
- for _, model := range models {
- t.Run(model, func(t *testing.T) {
- if time.Since(started) > softTimeout {
- t.Skip("skipping remaining tests to avoid excessive runtime")
- return
- }
- // Skip known-bad models unless explicitly requested via env var
- if reason, ok := skipModels[model]; ok && testModel == "" {
- t.Skipf("skipping: %s", reason)
- }
- if testModel != "" {
- requireCapability(ctx, t, client, model, "tools")
- }
- if v, ok := minVRAM[model]; ok {
- skipUnderMinVRAM(t, v)
- }
+ runAPIToolCallingStressModelWithClient(t, ctx, client, model, initialTimeout, streamTimeout, toolsMinVRAM, toolStressSkipModels)
+}
- pullOrSkip(ctx, t, client, model)
+func runAPIToolCallingStressModelWithClient(t *testing.T, ctx context.Context, client *api.Client, model string, initialTimeout, streamTimeout time.Duration, minVRAM map[string]uint64, skipModels map[string]string) {
+ t.Helper()
- // Preload and skip if not sufficiently GPU-loaded to avoid timeouts
- err := client.Generate(ctx, &api.GenerateRequest{Model: model}, func(response api.GenerateResponse) error { return nil })
- if err != nil {
- t.Fatalf("failed to load model %s: %s", model, err)
- }
- skipIfNotGPULoaded(ctx, t, client, model, 80)
-
- tools := stressTestTools()
-
- // Large system prompt that mimics real coding agents (opencode, Claude Code, etc.)
- // This is intentionally very long (~5000+ tokens) to match the prompt sizes that
- // real coding agents send. The combination of a large system prompt, many tools,
- // and thinking mode is what triggers failures in some models.
- systemPrompt := stressTestSystemPrompt()
-
- // Test 1: First request (fresh prompt processing)
- // Use a direct prompt that tells the model exactly what tool to use,
- // reducing the chance it asks for clarification instead.
- t.Run("first_request", func(t *testing.T) {
- testToolCall(t, ctx, client, model, systemPrompt, tools,
- "Run git diff main to review the code changes on the current branch.",
- initialTimeout, streamTimeout)
- })
-
- // Test 2: Repeat with same prompt (tests cache reuse)
- t.Run("cached_request", func(t *testing.T) {
- testToolCall(t, ctx, client, model, systemPrompt, tools,
- "Run git diff main to review the code changes on the current branch.",
- initialTimeout, streamTimeout)
- })
-
- // Test 3: Different user message (partial cache hit)
- t.Run("different_user_message", func(t *testing.T) {
- testToolCall(t, ctx, client, model, systemPrompt, tools,
- "Read the file at ./go.mod and tell me what dependencies we have.",
- initialTimeout, streamTimeout)
- })
-
- // Test 4: Multi-turn with tool response
- t.Run("multi_turn", func(t *testing.T) {
- testToolCallMultiTurn(t, ctx, client, model, systemPrompt, tools,
- initialTimeout, streamTimeout)
- })
- })
+ // Skip known-bad models unless explicitly requested via env var
+ if reason, ok := skipModels[model]; ok && testModel == "" {
+ t.Skipf("skipping: %s", reason)
+ }
+ if v, ok := minVRAM[model]; ok {
+ skipUnderMinVRAM(t, v)
}
+ requireCapability(ctx, t, client, model, "tools")
+
+ // Preload and skip if not sufficiently GPU-loaded to avoid timeouts
+ preloadGenerateModel(ctx, t, client, api.GenerateRequest{Model: model})
+ skipIfNotGPULoaded(ctx, t, client, model, 80)
+
+ tools := stressTestTools()
+
+ // Large system prompt that mimics real coding agents (opencode, Claude Code, etc.)
+ // This is intentionally very long (~5000+ tokens) to match the prompt sizes that
+ // real coding agents send. The combination of a large system prompt, many tools,
+ // and thinking mode is what triggers failures in some models.
+ systemPrompt := stressTestSystemPrompt()
+
+ // Test 1: First request (fresh prompt processing)
+ // Use a direct prompt that tells the model exactly what tool to use,
+ // reducing the chance it asks for clarification instead.
+ t.Run("first_request", func(t *testing.T) {
+ testToolCall(t, ctx, client, model, systemPrompt, tools,
+ "Run git diff main to review the code changes on the current branch.",
+ initialTimeout, streamTimeout)
+ })
+
+ // Test 2: Repeat with same prompt (tests cache reuse)
+ t.Run("cached_request", func(t *testing.T) {
+ testToolCall(t, ctx, client, model, systemPrompt, tools,
+ "Run git diff main to review the code changes on the current branch.",
+ initialTimeout, streamTimeout)
+ })
+
+ // Test 3: Different user message (partial cache hit)
+ t.Run("different_user_message", func(t *testing.T) {
+ testToolCall(t, ctx, client, model, systemPrompt, tools,
+ "Read the file at ./go.mod and tell me what dependencies we have.",
+ initialTimeout, streamTimeout)
+ })
+
+ // Test 4: Multi-turn with tool response
+ t.Run("multi_turn", func(t *testing.T) {
+ skipKnownIntegrationFlake(t, "tools-stress/multi_turn", model)
+ testToolCallMultiTurn(t, ctx, client, model, systemPrompt, tools,
+ initialTimeout, streamTimeout)
+ })
}
func newTool(name, description string, required []string, props map[string]api.ToolProperty) api.Tool {
diff --git a/integration/tools_test.go b/integration/tools_test.go
index 676c8c855..8fb4067e9 100644
--- a/integration/tools_test.go
+++ b/integration/tools_test.go
@@ -20,134 +20,138 @@ func testPropsMap(m map[string]api.ToolProperty) *api.ToolPropertiesMap {
return props
}
-func TestAPIToolCalling(t *testing.T) {
+func registerToolCases(models []string) {
+ registerModelIntegrationCases("tools", models, runAPIToolCallingModel)
+}
+
+var toolsMinVRAM = map[string]uint64{
+ "gemma4": 8,
+ "lfm2.5": 6,
+ "granite4.1:3b": 4,
+ "granite4.1:8b": 6,
+ "nemotron3:33b": 32,
+ "qwen3.5:2b": 4,
+ "qwen3.6:27b": 20,
+ "qwen3-vl": 16,
+ "gpt-oss:20b": 16,
+ "gpt-oss:120b": 70,
+ "qwen3": 6,
+ "llama3.1": 8,
+ "llama3.2": 4,
+ "mistral": 6,
+ "qwen2.5": 6,
+ "qwen2": 6,
+ "ministral-3": 20,
+ "mistral-nemo": 9,
+ "mistral-small": 16,
+ "mixtral:8x22b": 80,
+ "qwq": 20,
+ "granite3.3": 7,
+}
+
+func runAPIToolCallingModel(t *testing.T, model string) {
initialTimeout := 60 * time.Second
streamTimeout := 60 * time.Second
softTimeout, hardTimeout := getTimeouts(t)
+ if time.Since(started) > softTimeout {
+ t.Skip("skipping remaining tests to avoid excessive runtime")
+ }
ctx, cancel := context.WithTimeout(context.Background(), hardTimeout)
defer cancel()
client, _, cleanup := InitServerConnection(ctx, t)
defer cleanup()
- minVRAM := map[string]uint64{
- "gemma4": 8,
- "qwen3-vl": 16,
- "gpt-oss:20b": 16,
- "gpt-oss:120b": 70,
- "qwen3": 6,
- "llama3.1": 8,
- "llama3.2": 4,
- "mistral": 6,
- "qwen2.5": 6,
- "qwen2": 6,
- "ministral-3": 20,
- "mistral-nemo": 9,
- "mistral-small": 16,
- "mixtral:8x22b": 80,
- "qwq": 20,
- "granite3.3": 7,
- }
+ runAPIToolCallingModelWithClient(t, ctx, client, model, initialTimeout, streamTimeout, toolsMinVRAM)
+}
- models := testModels(libraryToolsModels)
-
- for _, model := range models {
- t.Run(model, func(t *testing.T) {
- if time.Now().Sub(started) > softTimeout {
- t.Skip("skipping remaining tests to avoid excessive runtime")
- return
- }
-
- if testModel != "" {
- requireCapability(ctx, t, client, model, "tools")
- }
- if v, ok := minVRAM[model]; ok {
- skipUnderMinVRAM(t, v)
- }
-
- pullOrSkip(ctx, t, client, model)
-
- tools := []api.Tool{
- {
- Type: "function",
- Function: api.ToolFunction{
- Name: "get_weather",
- Description: "Get the current weather in a given location",
- Parameters: api.ToolFunctionParameters{
- Type: "object",
- Required: []string{"location"},
- Properties: testPropsMap(map[string]api.ToolProperty{
- "location": {
- Type: api.PropertyType{"string"},
- Description: "The city and state, e.g. San Francisco, CA",
- },
- }),
+func runAPIToolCallingModelWithClient(t *testing.T, ctx context.Context, client *api.Client, model string, initialTimeout, streamTimeout time.Duration, minVRAM map[string]uint64) {
+ t.Helper()
+
+ if v, ok := minVRAM[model]; ok {
+ skipUnderMinVRAM(t, v)
+ }
+ requireCapability(ctx, t, client, model, "tools")
+
+ tools := []api.Tool{
+ {
+ Type: "function",
+ Function: api.ToolFunction{
+ Name: "get_weather",
+ Description: "Get the current weather for a location",
+ Parameters: api.ToolFunctionParameters{
+ Type: "object",
+ Required: []string{"location"},
+ Properties: testPropsMap(map[string]api.ToolProperty{
+ "location": {
+ Type: api.PropertyType{"string"},
+ Description: "The city and state, e.g. San Francisco, CA",
},
- },
+ }),
},
- }
-
- req := api.ChatRequest{
- Model: model,
- Messages: []api.Message{
- {
- Role: "user",
- Content: "Call get_weather with location set to San Francisco.",
- },
- },
- Tools: tools,
- Options: map[string]any{
- "temperature": 0,
- },
- KeepAlive: &api.Duration{Duration: 10 * time.Second},
- }
-
- stallTimer := time.NewTimer(initialTimeout)
- var gotToolCall bool
- var lastToolCall api.ToolCall
-
- fn := func(response api.ChatResponse) error {
- if len(response.Message.ToolCalls) > 0 {
- gotToolCall = true
- lastToolCall = response.Message.ToolCalls[len(response.Message.ToolCalls)-1]
- }
- if !stallTimer.Reset(streamTimeout) {
- return fmt.Errorf("stall was detected while streaming response, aborting")
- }
- return nil
- }
-
- stream := true
- req.Stream = &stream
- done := make(chan int)
- var genErr error
- go func() {
- genErr = client.Chat(ctx, &req, fn)
- done <- 0
- }()
-
- select {
- case <-stallTimer.C:
- t.Errorf("tool-calling chat never started. Timed out after: %s", initialTimeout.String())
- case <-done:
- if genErr != nil {
- t.Fatalf("chat failed: %v", genErr)
- }
-
- if !gotToolCall {
- t.Fatalf("expected at least one tool call, got none")
- }
-
- if lastToolCall.Function.Name != "get_weather" {
- t.Errorf("unexpected tool called: got %q want %q", lastToolCall.Function.Name, "get_weather")
- }
-
- if _, ok := lastToolCall.Function.Arguments.Get("location"); !ok {
- t.Errorf("expected tool arguments to include 'location', got: %s", lastToolCall.Function.Arguments.String())
- }
- case <-ctx.Done():
- t.Error("outer test context done while waiting for tool-calling chat")
- }
- })
+ },
+ },
+ }
+
+ req := api.ChatRequest{
+ Model: model,
+ Messages: []api.Message{
+ {
+ Role: "user",
+ Content: "Call get_weather with location set to San Francisco.",
+ },
+ },
+ Tools: tools,
+ Options: map[string]any{
+ "temperature": 0,
+ },
+ KeepAlive: &api.Duration{Duration: 10 * time.Second},
+ }
+
+ stallTimer := time.NewTimer(initialTimeout)
+ var gotToolCall bool
+ var lastToolCall api.ToolCall
+
+ fn := func(response api.ChatResponse) error {
+ if len(response.Message.ToolCalls) > 0 {
+ gotToolCall = true
+ lastToolCall = response.Message.ToolCalls[len(response.Message.ToolCalls)-1]
+ }
+ if !stallTimer.Reset(streamTimeout) {
+ return fmt.Errorf("stall was detected while streaming response, aborting")
+ }
+ return nil
+ }
+
+ stream := true
+ req.Stream = &stream
+ done := make(chan int)
+ var genErr error
+ go func() {
+ genErr = client.Chat(ctx, &req, fn)
+ done <- 0
+ }()
+
+ select {
+ case <-stallTimer.C:
+ t.Errorf("tool-calling chat never started. Timed out after: %s", initialTimeout.String())
+ case <-done:
+ if genErr != nil {
+ t.Fatalf("chat failed: %v", genErr)
+ }
+
+ if !gotToolCall {
+ t.Fatalf("expected at least one tool call, got none")
+ }
+
+ if lastToolCall.Function.Name != "get_weather" {
+ t.Errorf("unexpected tool called: got %q want %q", lastToolCall.Function.Name, "get_weather")
+ }
+
+ if _, ok := lastToolCall.Function.Arguments.Get("location"); !ok {
+ t.Errorf("expected tool arguments to include 'location', got: %s", lastToolCall.Function.Arguments.String())
+ }
+ case <-ctx.Done():
+ t.Error("outer test context done while waiting for tool-calling chat")
}
}
diff --git a/integration/utils_test.go b/integration/utils_test.go
index b29ae8833..c5e4559c8 100644
--- a/integration/utils_test.go
+++ b/integration/utils_test.go
@@ -31,270 +31,18 @@ import (
)
var (
- smol = "llama3.2:1b"
- stream = false
-
// testModel is set via OLLAMA_TEST_MODEL env var. When set, all tests
// that loop over model lists will test only this model, and smol is
// also overridden to use it.
- testModel string
+ testModel = os.Getenv("OLLAMA_TEST_MODEL")
+
+ smol = defaultTestModel("llama3.2:1b")
+ stream = false
)
var (
started = time.Now()
- // Note: add newer models at the top of the list to test them first
- ollamaEngineChatModels = []string{
- "nemotron3:33b",
- // "laguna-xs.2:q4_K_M", // TODO: re-enable when llama.cpp supports laguna.
- "gemma4",
- "lfm2.5-thinking",
- "ministral-3",
- "qwen3-coder:30b",
- "gpt-oss:20b",
- "gemma3n:e2b",
- "mistral-small3.2:latest",
- "deepseek-r1:1.5b",
- // "llama3.2-vision:latest", // TODO: re-enable when llama.cpp supports mllama.
- "qwen2.5-coder:latest",
- "qwen2.5vl:3b",
- "qwen3:0.6b", // dense
- "qwen3:1.7b", // dense
- "qwen3:30b", // MOE
- "gemma3:1b",
- "llama3.1:latest",
- "llama3.2:latest",
- "gemma2:latest",
- "minicpm-v:latest", // arch=qwen2
- "granite-code:latest", // arch=llama
- }
- // MLX-backed safetensors tags. These exercise the mlxrunner subprocess
- // on platforms where MLX is available (today: macOS; Linux/Windows CUDA
- // coming). On other platforms, skipIfMLXUnsupported turns the load
- // failure into a test skip.
- mlxEngineChatModels = []string{
- "laguna-xs.2:nvfp4",
- "qwen3.5:2b-nvfp4", // ~2.5GB, Qwen3_5 arch
- "gemma4:e2b-nvfp4", // ~7.1GB, Gemma4 arch (skipped under low VRAM)
- }
- llamaRunnerChatModels = []string{
- "mistral:latest",
- "falcon3:latest",
- "granite3-moe:latest",
- "command-r:latest",
- "nemotron-mini:latest",
- "phi3.5:latest",
- "internlm2:latest",
- "codellama:latest", // arch=llama
- "phi3:latest",
- }
-
- // Some library models are quite large - ensure large VRAM and sufficient disk space
- // before running scenarios based on this set
- libraryChatModels = []string{
- "alfred",
- "athene-v2",
- "aya-expanse",
- "aya",
- "bakllava",
- "bespoke-minicheck",
- "codebooga",
- "codegeex4",
- "codegemma",
- "codellama",
- "codeqwen",
- "codestral",
- "codeup",
- "cogito",
- "command-a",
- "command-r-plus",
- "command-r",
- "command-r7b-arabic",
- "command-r7b",
- "dbrx",
- "deepcoder",
- "deepscaler",
- "deepseek-coder-v2",
- "deepseek-coder",
- "deepseek-llm",
- "deepseek-r1",
- // "deepseek-v2.5", // requires 155 GB VRAM
- "deepseek-v2",
- // "deepseek-v3", // requires 482 GB VRAM
- "devstral",
- "dolphin-llama3",
- "dolphin-mistral",
- "dolphin-mixtral",
- "dolphin-phi",
- "dolphin3",
- "dolphincoder",
- "duckdb-nsql",
- "everythinglm",
- "exaone-deep",
- "exaone3.5",
- "falcon",
- "falcon2",
- "falcon3",
- "firefunction-v2",
- "gemma",
- "gemma2",
- "gemma3",
- "gemma3n",
- "gemma4",
- "glm4",
- "goliath",
- "gpt-oss:20b",
- "granite-code",
- "granite3-dense",
- "granite3-guardian",
- "granite3-moe",
- "granite3.1-dense",
- "granite3.1-moe",
- "granite3.2-vision",
- "granite3.2",
- "granite3.3",
- "hermes3",
- "internlm2",
- "lfm2.5-thinking",
- "llama-guard3",
- "llama-pro",
- "llama2-chinese",
- "llama2-uncensored",
- "llama2",
- "llama3-chatqa",
- "llama3-gradient",
- "llama3-groq-tool-use",
- "llama3.1",
- // "llama3.2-vision", // TODO: re-enable when llama.cpp supports mllama.
- "llama3.2",
- "llama3.3",
- "llama3",
- "llama4",
- "llava-llama3",
- "llava-phi3",
- "llava",
- "magicoder",
- "magistral",
- "marco-o1",
- "mathstral",
- "meditron",
- "medllama2",
- "megadolphin",
- "minicpm-v",
- "ministral-3",
- "mistral-large",
- "mistral-nemo",
- "mistral-openorca",
- "mistral-small",
- "mistral-small3.1",
- "mistral-small3.2",
- "mistral",
- "mistrallite",
- "mixtral",
- "moondream",
- "nemotron-mini",
- "nemotron",
- "neural-chat",
- "nexusraven",
- "notus",
- "nous-hermes",
- "nous-hermes2-mixtral",
- "nous-hermes2",
- "nuextract",
- "olmo2",
- "open-orca-platypus2",
- "openchat",
- "opencoder",
- "openhermes",
- "openthinker",
- "orca-mini",
- "orca2",
- // "phi", // unreliable
- "phi3.5",
- "phi3",
- "phi4-mini-reasoning",
- "phi4-mini",
- "phi4-reasoning",
- "phi4",
- "phind-codellama",
- "qwen",
- "qwen2-math",
- "qwen2.5-coder",
- "qwen2.5",
- "qwen2.5vl",
- "qwen2",
- "qwen3:0.6b", // dense
- "qwen3:30b", // MOE
- "qwq",
- "r1-1776",
- "reader-lm",
- "reflection",
- "sailor2",
- "samantha-mistral",
- "shieldgemma",
- "smallthinker",
- "smollm",
- "smollm2",
- "solar",
- "sqlcoder",
- "stable-beluga",
- "stable-code",
- "stablelm-zephyr",
- "stablelm2",
- "starcoder",
- "starcoder2",
- "starling-lm",
- "tinydolphin",
- "tinyllama",
- "tulu3",
- "vicuna",
- "wizard-math",
- "wizard-vicuna-uncensored",
- "wizard-vicuna",
- "wizardcoder",
- "wizardlm-uncensored",
- "wizardlm2",
- "xwinlm",
- "yarn-llama2",
- "yarn-mistral",
- "yi-coder",
- "yi",
- "zephyr",
- }
- libraryEmbedModels = []string{
- "embeddinggemma",
- "nomic-embed-text",
- "all-minilm",
- "bge-large",
- "bge-m3",
- "granite-embedding",
- "mxbai-embed-large",
- "paraphrase-multilingual",
- "snowflake-arctic-embed",
- "snowflake-arctic-embed2",
- "qwen3-embedding",
- }
- libraryToolsModels = []string{
- "nemotron3:33b",
- // "laguna-xs.2", // TODO: re-enable when llama.cpp supports laguna.
- "gemma4",
- "lfm2.5-thinking",
- "qwen3-vl",
- "gpt-oss:20b",
- "gpt-oss:120b",
- "qwen3",
- "llama3.1",
- "llama3.2",
- "mistral",
- "qwen2.5",
- "ministral-3",
- "mistral-nemo",
- "mistral-small",
- "mixtral:8x22b",
- "qwq",
- "granite3.3",
- }
-
blueSkyPrompt = "why is the sky blue? Be brief but factual in your reply"
blueSkyExpected = []string{"rayleigh", "scatter", "atmosphere", "nitrogen", "oxygen", "wavelength", "interact"}
@@ -314,13 +62,18 @@ func init() {
logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}))
slog.SetDefault(logger)
- testModel = os.Getenv("OLLAMA_TEST_MODEL")
if testModel != "" {
slog.Info("test model override", "model", testModel)
- smol = testModel
}
}
+func defaultTestModel(model string) string {
+ if testModel != "" {
+ return testModel
+ }
+ return model
+}
+
// testModels returns the override model as a single-element slice when
// OLLAMA_TEST_MODEL is set, otherwise returns the provided default list.
func testModels(defaults []string) []string {
@@ -500,7 +253,7 @@ func PullIfMissing(ctx context.Context, client *api.Client, modelName string) er
}
slog.Info("model missing", "model", modelName)
- stallDuration := 60 * time.Second // This includes checksum verification, which can take a while on larger models, and slower systems
+ stallDuration := 2 * time.Minute // Includes checksum verification, which can take a while on larger models and slower systems.
stallTimer := time.NewTimer(stallDuration)
fn := func(resp api.ProgressResponse) error {
// fmt.Print(".")
@@ -632,14 +385,7 @@ func DoGenerate(ctx context.Context, t *testing.T, client *api.Client, genReq ap
verify := func() {
// Verify the response contains the expected data
response = buf.String()
- atLeastOne := false
- for _, resp := range anyResp {
- if strings.Contains(strings.ToLower(response), resp) {
- atLeastOne = true
- break
- }
- }
- if !atLeastOne {
+ if !containsExpectedResponse(response, anyResp) {
t.Fatalf("%s: none of %v found in %s", genReq.Model, anyResp, response)
}
}
@@ -777,14 +523,7 @@ func DoChat(ctx context.Context, t *testing.T, client *api.Client, req api.ChatR
verify := func() {
// Verify the response contains the expected data
response = buf.String()
- atLeastOne := false
- for _, resp := range anyResp {
- if strings.Contains(strings.ToLower(response), resp) {
- atLeastOne = true
- break
- }
- }
- if !atLeastOne {
+ if !containsExpectedResponse(response, anyResp) {
t.Fatalf("%s: none of %v found in \"%s\" -- request was:%s", req.Model, anyResp, response, summarizeMessages(req.Messages))
}
}
@@ -816,6 +555,24 @@ func DoChat(ctx context.Context, t *testing.T, client *api.Client, req api.ChatR
return &api.Message{Role: role, Content: buf.String()}
}
+func containsExpectedResponse(response string, anyResp []string) bool {
+ lowerResponse := strings.ToLower(response)
+ normalizedResponse := normalizeResponseText(response)
+ for _, resp := range anyResp {
+ if strings.Contains(lowerResponse, strings.ToLower(resp)) {
+ return true
+ }
+ if strings.Contains(normalizedResponse, normalizeResponseText(resp)) {
+ return true
+ }
+ }
+ return false
+}
+
+func normalizeResponseText(s string) string {
+ return strings.Join(strings.Fields(strings.ToLower(s)), " ")
+}
+
func ChatRequests() ([]api.ChatRequest, [][]string) {
genReqs, results := GenerateRequests()
reqs := make([]api.ChatRequest, len(genReqs))
@@ -835,6 +592,16 @@ func ChatRequests() ([]api.ChatRequest, [][]string) {
return reqs, results
}
+func preloadGenerateModel(ctx context.Context, t *testing.T, client *api.Client, req api.GenerateRequest) {
+ t.Helper()
+ slog.Info("loading", "model", req.Model)
+ err := client.Generate(ctx, &req, func(response api.GenerateResponse) error { return nil })
+ if err != nil {
+ skipIfMLXUnsupported(t, err)
+ t.Fatalf("failed to load model %s: %s", req.Model, err)
+ }
+}
+
// skipIfMLXUnsupported converts an MLX runner startup error into a test skip
// when the fingerprint matches "the MLX stack is not wired up on this host",
// and only on platforms where MLX is not yet expected to work. On Apple
@@ -851,7 +618,8 @@ func skipIfMLXUnsupported(t *testing.T, err error) {
if err == nil {
return
}
- if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
+ targetGOOS, targetGOARCH := targetPlatform()
+ if targetGOOS == "darwin" && targetGOARCH == "arm64" {
return
}
msg := err.Error()
@@ -863,17 +631,52 @@ func skipIfMLXUnsupported(t *testing.T, err error) {
"image generation is not supported on",
} {
if strings.Contains(msg, s) {
- t.Skipf("MLX not available on %s/%s: %v", runtime.GOOS, runtime.GOARCH, err)
+ t.Skipf("MLX not available on target %s/%s (runner %s/%s): %v", targetGOOS, targetGOARCH, runtime.GOOS, runtime.GOARCH, err)
}
}
}
+func targetPlatform() (goos, goarch string) {
+ goos = normalizeTargetGOOS(os.Getenv("OLLAMA_TEST_HOST_OS"))
+ goarch = normalizeTargetGOARCH(os.Getenv("OLLAMA_TEST_HOST_ARCH"))
+ if goos == "" {
+ goos = runtime.GOOS
+ }
+ if goarch == "" {
+ goarch = runtime.GOARCH
+ }
+ return goos, goarch
+}
+
+func normalizeTargetGOOS(goos string) string {
+ switch strings.ToLower(goos) {
+ case "darwin":
+ return "darwin"
+ case "linux":
+ return "linux"
+ case "windows", "win32nt":
+ return "windows"
+ default:
+ return strings.ToLower(goos)
+ }
+}
+
+func normalizeTargetGOARCH(goarch string) string {
+ switch strings.ToLower(goarch) {
+ case "aarch64", "arm64":
+ return "arm64"
+ case "x86_64", "amd64":
+ return "amd64"
+ default:
+ return strings.ToLower(goarch)
+ }
+}
+
// skipIfModelTooLargeForVRAM skips the test when the model's on-disk size
// is larger than OLLAMA_MAX_VRAM by enough that even partial GPU offload
-// won't help. Uses the same 0.75x gate as TestPerfModels (model_perf_test.go)
-// so vision/audio tests stay runnable on systems where the model is slightly
-// over VRAM and a portion legitimately spills to CPU. No-op when
-// OLLAMA_MAX_VRAM is unset.
+// won't help. The 0.75x gate keeps vision/audio tests runnable on systems
+// where the model is slightly over VRAM and a portion legitimately spills to
+// CPU. No-op when OLLAMA_MAX_VRAM is unset.
func skipIfModelTooLargeForVRAM(ctx context.Context, t *testing.T, client *api.Client, modelName string) {
t.Helper()
s := os.Getenv("OLLAMA_MAX_VRAM")
diff --git a/integration/vision_test.go b/integration/vision_test.go
index eba071112..44903c7ba 100644
--- a/integration/vision_test.go
+++ b/integration/vision_test.go
@@ -13,17 +13,6 @@ import (
"github.com/ollama/ollama/types/model"
)
-// Default set of vision models to test. When OLLAMA_TEST_MODEL is set,
-// only that model is tested (with a capability check for vision).
-var defaultVisionModels = []string{
- "nemotron3:33b",
- "gemma4",
- "gemma3",
- // "llama3.2-vision", // TODO: re-enable when llama.cpp supports mllama.
- "qwen2.5vl",
- "qwen3-vl:8b",
-}
-
// decodeTestImages returns the test images.
func decodeTestImages(t *testing.T) (abbeyRoad, docs, ollamaHome api.ImageData) {
t.Helper()
@@ -68,22 +57,17 @@ func skipIfNoVisionOverride(t *testing.T) {
// setupVisionModel pulls the model, preloads it, and skips if not GPU-loaded.
func setupVisionModel(ctx context.Context, t *testing.T, client *api.Client, model string) {
t.Helper()
- if testModel == "" {
- pullOrSkip(ctx, t, client, model)
- }
+ pullOrSkip(ctx, t, client, model)
skipIfModelTooLargeForVRAM(ctx, t, client, model)
requireCapability(ctx, t, client, model, "vision")
- err := client.Generate(ctx, &api.GenerateRequest{Model: model}, func(response api.GenerateResponse) error { return nil })
- if err != nil {
- t.Fatalf("failed to load model %s: %s", model, err)
- }
+ preloadGenerateModel(ctx, t, client, api.GenerateRequest{Model: model})
skipIfNotGPULoaded(ctx, t, client, model, 80)
}
-// TestVisionMultiTurn sends an image, gets a response, then asks follow-up
+// runVisionMultiTurn sends an image, gets a response, then asks follow-up
// questions about the same image. This verifies that the KV cache correctly
// handles cached image tokens across turns.
-func TestVisionMultiTurn(t *testing.T) {
+func runVisionMultiTurn(t *testing.T, models []string) {
skipUnderMinVRAM(t, 16)
skipIfNoVisionOverride(t)
@@ -93,8 +77,9 @@ func TestVisionMultiTurn(t *testing.T) {
"llama3.2-vision": "miscounts animals (says 3 instead of 4) on turn 2",
}
- for _, model := range testModels(defaultVisionModels) {
+ for _, model := range testModels(models) {
t.Run(model, func(t *testing.T) {
+ skipKnownIntegrationFlake(t, "vision-multiturn", model)
if reason, ok := skipModels[model]; ok && testModel == "" {
t.Skipf("skipping: %s", reason)
}
@@ -151,8 +136,8 @@ func TestVisionMultiTurn(t *testing.T) {
}
}
-// TestVisionObjectCounting asks the model to count objects in an image.
-func TestVisionObjectCounting(t *testing.T) {
+// runVisionObjectCounting asks the model to count objects in an image.
+func runVisionObjectCounting(t *testing.T, models []string) {
skipUnderMinVRAM(t, 16)
skipIfNoVisionOverride(t)
@@ -160,8 +145,9 @@ func TestVisionObjectCounting(t *testing.T) {
"llama3.2-vision": "consistently miscounts (says 3 instead of 4)",
}
- for _, model := range testModels(defaultVisionModels) {
+ for _, model := range testModels(models) {
t.Run(model, func(t *testing.T) {
+ skipKnownIntegrationFlake(t, "vision-count", model)
if reason, ok := skipModels[model]; ok && testModel == "" {
t.Skipf("skipping: %s", reason)
}
@@ -191,9 +177,9 @@ func TestVisionObjectCounting(t *testing.T) {
}
}
-// TestVisionSceneUnderstanding tests whether the model can identify
+// runVisionSceneUnderstanding tests whether the model can identify
// cultural references and scene context from an image.
-func TestVisionSceneUnderstanding(t *testing.T) {
+func runVisionSceneUnderstanding(t *testing.T, models []string) {
skipUnderMinVRAM(t, 16)
skipIfNoVisionOverride(t)
@@ -203,7 +189,7 @@ func TestVisionSceneUnderstanding(t *testing.T) {
"minicpm-v": "too small for cultural reference detection",
}
- for _, model := range testModels(defaultVisionModels) {
+ for _, model := range testModels(models) {
t.Run(model, func(t *testing.T) {
if reason, ok := skipModels[model]; ok && testModel == "" {
t.Skipf("skipping: %s", reason)
@@ -236,13 +222,13 @@ func TestVisionSceneUnderstanding(t *testing.T) {
}
}
-// TestVisionSpatialReasoning tests the model's ability to identify
+// runVisionSpatialReasoning tests the model's ability to identify
// objects based on their spatial position in the image.
-func TestVisionSpatialReasoning(t *testing.T) {
+func runVisionSpatialReasoning(t *testing.T, models []string) {
skipUnderMinVRAM(t, 16)
skipIfNoVisionOverride(t)
- for _, model := range testModels(defaultVisionModels) {
+ for _, model := range testModels(models) {
t.Run(model, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
@@ -274,13 +260,13 @@ func TestVisionSpatialReasoning(t *testing.T) {
}
}
-// TestVisionDetailRecognition tests whether the model can identify
+// runVisionDetailRecognition tests whether the model can identify
// small details like accessories in an image.
-func TestVisionDetailRecognition(t *testing.T) {
+func runVisionDetailRecognition(t *testing.T, models []string) {
skipUnderMinVRAM(t, 16)
skipIfNoVisionOverride(t)
- for _, model := range testModels(defaultVisionModels) {
+ for _, model := range testModels(models) {
t.Run(model, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
@@ -310,10 +296,10 @@ func TestVisionDetailRecognition(t *testing.T) {
}
}
-// TestVisionMultiImage sends two images in a single message and asks
+// runVisionMultiImage sends two images in a single message and asks
// the model to compare and contrast them. This exercises multi-image
// encoding and cross-image reasoning.
-func TestVisionMultiImage(t *testing.T) {
+func runVisionMultiImage(t *testing.T, models []string) {
skipUnderMinVRAM(t, 16)
skipIfNoVisionOverride(t)
@@ -322,7 +308,7 @@ func TestVisionMultiImage(t *testing.T) {
"llama3.2-vision": "does not support multi-image input",
}
- for _, model := range testModels(defaultVisionModels) {
+ for _, model := range testModels(models) {
t.Run(model, func(t *testing.T) {
if reason, ok := skipModels[model]; ok && testModel == "" {
t.Skipf("skipping: %s", reason)
@@ -357,14 +343,14 @@ func TestVisionMultiImage(t *testing.T) {
}
}
-// TestVisionImageDescription verifies that the model can describe the contents
+// runVisionImageDescription verifies that the model can describe the contents
// of the ollama homepage image (a cartoon llama with "Start building with
// open models" text). Basic sanity check that the vision pipeline works.
-func TestVisionImageDescription(t *testing.T) {
+func runVisionImageDescription(t *testing.T, models []string) {
skipUnderMinVRAM(t, 16)
skipIfNoVisionOverride(t)
- for _, model := range testModels(defaultVisionModels) {
+ for _, model := range testModels(models) {
t.Run(model, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()