Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
tests: add single threaded history test (#12295)
Daniel Hiltgenc23e6f4cae3c
diff --git a/integration/README.md b/integration/README.mdindex e52ba71ee..1dfd0e359 100644--- a/integration/README.md+++ b/integration/README.md@@ -12,3 +12,6 @@ The integration tests have 2 modes of operating.> [!IMPORTANT]> Before running the tests locally without the "test existing" setting, compile ollama from the top of the source tree `go build .` in addition to GPU support with cmake if applicable on your platform. The integration tests expect to find an ollama binary at the top of the tree.+++Many tests use a default small model suitable to run on many systems. You can override this default model by setting `OLLAMA_TEST_DEFAULT_MODEL`\ No newline at end of filediff --git a/integration/api_test.go b/integration/api_test.goindex c39192c99..48572085d 100644--- a/integration/api_test.go+++ b/integration/api_test.go@@ -22,13 +22,12 @@ func TestAPIGenerate(t *testing.T) {// Set up the test datareq := api.GenerateRequest{Model: smol,- Prompt: "why is the sky blue? be brief",+ Prompt: blueSkyPrompt,Options: map[string]interface{}{"temperature": 0,"seed": 123,},}- anyResp := []string{"rayleigh", "scattering"}client, _, cleanup := InitServerConnection(ctx, t)defer cleanup()@@ -120,14 +119,14 @@ func TestAPIGenerate(t *testing.T) {// Verify the response contains the expected dataresponse := buf.String()atLeastOne := false- for _, resp := range anyResp {+ for _, resp := range blueSkyExpected {if strings.Contains(strings.ToLower(response), resp) {atLeastOne = truebreak}}if !atLeastOne {- t.Errorf("none of %v found in %s", anyResp, response)+ t.Errorf("none of %v found in %s", blueSkyExpected, response)}case <-ctx.Done():t.Error("outer test context done while waiting for generate")@@ -181,7 +180,7 @@ func TestAPIChat(t *testing.T) {Messages: []api.Message{{Role: "user",- Content: "why is the sky blue? be brief",+ Content: blueSkyPrompt,},},Options: map[string]interface{}{@@ -189,7 +188,6 @@ func TestAPIChat(t *testing.T) {"seed": 123,},}- anyResp := []string{"rayleigh", "scattering"}client, _, cleanup := InitServerConnection(ctx, t)defer cleanup()@@ -279,14 +277,14 @@ func TestAPIChat(t *testing.T) {// Verify the response contains the expected dataresponse := buf.String()atLeastOne := false- for _, resp := range anyResp {+ for _, resp := range blueSkyExpected {if strings.Contains(strings.ToLower(response), resp) {atLeastOne = truebreak}}if !atLeastOne {- t.Errorf("none of %v found in %s", anyResp, response)+ t.Errorf("none of %v found in %s", blueSkyExpected, response)}case <-ctx.Done():t.Error("outer test context done while waiting for chat")diff --git a/integration/basic_test.go b/integration/basic_test.goindex 60cff172b..0a6b9253d 100644--- a/integration/basic_test.go+++ b/integration/basic_test.go@@ -19,14 +19,14 @@ func TestBlueSky(t *testing.T) {// Set up the test datareq := api.GenerateRequest{Model: smol,- Prompt: "why is the sky blue?",+ Prompt: blueSkyPrompt,Stream: &stream,Options: map[string]any{"temperature": 0,"seed": 123,},}- GenerateTestHelper(ctx, t, req, []string{"rayleigh", "scattering"})+ GenerateTestHelper(ctx, t, req, blueSkyExpected)}func TestUnicode(t *testing.T) {@@ -110,12 +110,12 @@ func TestUnicodeModelDir(t *testing.T) {req := api.GenerateRequest{Model: smol,- Prompt: "why is the sky blue?",+ Prompt: blueSkyPrompt,Stream: &stream,Options: map[string]any{"temperature": 0,"seed": 123,},}- GenerateTestHelper(ctx, t, req, []string{"rayleigh", "scattering"})+ GenerateTestHelper(ctx, t, req, blueSkyExpected)}diff --git a/integration/context_test.go b/integration/context_test.goindex 15c157858..9d13f7acb 100644--- a/integration/context_test.go+++ b/integration/context_test.go@@ -63,11 +63,11 @@ func TestContextExhaustion(t *testing.T) {if err := PullIfMissing(ctx, client, req.Model); err != nil {t.Fatalf("PullIfMissing failed: %v", err)}- DoGenerate(ctx, t, client, req, []string{"once", "upon", "lived", "sunny", "cloudy", "clear", "water"}, 120*time.Second, 10*time.Second)+ DoGenerate(ctx, t, client, req, []string{"once", "upon", "lived", "sunny", "cloudy", "clear", "water", "time", "travel", "world"}, 120*time.Second, 10*time.Second)}// Send multiple generate requests with prior context and ensure the response is coherant and expected-func TestGenerateWithHistory(t *testing.T) {+func TestParallelGenerateWithHistory(t *testing.T) {modelOverride := ollamaEngineChatModels[0] // Most recent ollama engine modelreq, resp := GenerateRequests()numParallel := 2@@ -113,8 +113,48 @@ func TestGenerateWithHistory(t *testing.T) {wg.Wait()}+// Send generate requests with prior context and ensure the response is coherant and expected+func TestGenerateWithHistory(t *testing.T) {+ req := api.GenerateRequest{+ Model: smol,+ Prompt: rainbowPrompt,+ Stream: &stream,+ KeepAlive: &api.Duration{Duration: 10 * time.Second},+ Options: map[string]any{+ "num_ctx": 16384,+ },+ }++ softTimeout, hardTimeout := getTimeouts(t)+ ctx, cancel := context.WithTimeout(context.Background(), hardTimeout)+ defer cancel()+ client, _, cleanup := InitServerConnection(ctx, t)+ defer cleanup()++ // Get the server running (if applicable) warm the model up with a single initial request+ slog.Info("loading", "model", req.Model)+ err := client.Generate(ctx,+ &api.GenerateRequest{Model: req.Model, KeepAlive: &api.Duration{Duration: 10 * time.Second}, Options: req.Options},+ func(response api.GenerateResponse) error { return nil },+ )+ if err != nil {+ t.Fatalf("failed to load model %s: %s", req.Model, err)+ }++ req.Context = DoGenerate(ctx, t, client, req, rainbowExpected, 30*time.Second, 20*time.Second)++ for i := 0; i < len(rainbowFollowups); i++ {+ req.Prompt = rainbowFollowups[i]+ if time.Now().Sub(started) > softTimeout {+ slog.Info("exceeded soft timeout, winding down test")+ return+ }+ req.Context = DoGenerate(ctx, t, client, req, rainbowExpected, 30*time.Second, 20*time.Second)+ }+}+// Send multiple chat requests with prior context and ensure the response is coherant and expected-func TestChatWithHistory(t *testing.T) {+func TestParallelChatWithHistory(t *testing.T) {modelOverride := ollamaEngineChatModels[0] // Most recent ollama engine modelreq, resp := ChatRequests()numParallel := 2@@ -164,3 +204,55 @@ func TestChatWithHistory(t *testing.T) {}wg.Wait()}++// Send generate requests with prior context and ensure the response is coherant and expected+func TestChatWithHistory(t *testing.T) {+ req := api.ChatRequest{+ Model: smol,+ Stream: &stream,+ KeepAlive: &api.Duration{Duration: 10 * time.Second},+ Options: map[string]any{+ "num_ctx": 16384,+ },+ Messages: []api.Message{+ {+ Role: "user",+ Content: rainbowPrompt,+ },+ },+ }++ softTimeout, hardTimeout := getTimeouts(t)+ ctx, cancel := context.WithTimeout(context.Background(), hardTimeout)+ defer cancel()+ client, _, cleanup := InitServerConnection(ctx, t)+ defer cleanup()++ // Get the server running (if applicable) warm the model up with a single initial request+ slog.Info("loading", "model", req.Model)+ err := client.Generate(ctx,+ &api.GenerateRequest{Model: req.Model, KeepAlive: &api.Duration{Duration: 10 * time.Second}, Options: req.Options},+ func(response api.GenerateResponse) error { return nil },+ )+ if err != nil {+ t.Fatalf("failed to load model %s: %s", req.Model, err)+ }++ assistant := DoChat(ctx, t, client, req, rainbowExpected, 30*time.Second, 20*time.Second)++ for i := 0; i < len(rainbowFollowups); i++ {+ if time.Now().Sub(started) > softTimeout {+ slog.Info("exceeded soft timeout, winding down test")+ return+ }+ req.Messages = append(req.Messages,+ *assistant,+ api.Message{Role: "user", Content: rainbowFollowups[i]},+ )++ assistant = DoChat(ctx, t, client, req, rainbowExpected, 30*time.Second, 20*time.Second)+ if assistant == nil {+ t.Fatalf("didn't get an assistant response for context")+ }+ }+}diff --git a/integration/library_models_test.go b/integration/library_models_test.goindex cdf65efc8..49e1097b8 100644--- a/integration/library_models_test.go+++ b/integration/library_models_test.go@@ -4,7 +4,9 @@ package integrationimport ("context"+ "fmt""log/slog"+ "os""testing""time"@@ -20,6 +22,7 @@ func TestLibraryModelsGenerate(t *testing.T) {defer cancel()client, _, cleanup := InitServerConnection(ctx, t)defer cleanup()+ targetArch := os.Getenv("OLLAMA_TEST_ARCHITECTURE")chatModels := libraryChatModelsfor _, model := range chatModels {@@ -30,16 +33,26 @@ func TestLibraryModelsGenerate(t *testing.T) {if err := PullIfMissing(ctx, client, model); err != nil {t.Fatalf("pull failed %s", err)}+ if targetArch != "" {+ resp, err := client.Show(ctx, &api.ShowRequest{Name: model})+ if err != nil {+ t.Fatalf("unable to show model: %s", err)+ }+ arch := resp.ModelInfo["general.architecture"].(string)+ if arch != targetArch {+ t.Skip(fmt.Sprintf("Skipping %s architecture %s != %s", model, arch, targetArch))+ }+ }req := api.GenerateRequest{Model: model,- Prompt: "why is the sky blue?",+ Prompt: blueSkyPrompt,KeepAlive: &api.Duration{Duration: 10 * time.Second},Options: map[string]interface{}{"temperature": 0.1,"seed": 123,},}- anyResp := []string{"rayleigh", "scatter", "atmosphere", "nitrogen", "oxygen", "wavelength"}+ anyResp := blueSkyExpected// Special casesif model == "duckdb-nsql" {anyResp = []string{"select", "from"}diff --git a/integration/model_arch_test.go b/integration/model_arch_test.goindex 9fc2e01dd..721d95c54 100644--- a/integration/model_arch_test.go+++ b/integration/model_arch_test.go@@ -68,14 +68,13 @@ func TestModelsGenerate(t *testing.T) {// TODO - fiddle with context sizereq := api.GenerateRequest{Model: model,- Prompt: "why is the sky blue?",+ Prompt: blueSkyPrompt,Options: map[string]interface{}{"temperature": 0,"seed": 123,},}- anyResp := []string{"rayleigh", "scattering", "atmosphere", "nitrogen", "oxygen"}- DoGenerate(ctx, t, client, req, anyResp, 120*time.Second, 30*time.Second)+ DoGenerate(ctx, t, client, req, blueSkyExpected, 120*time.Second, 30*time.Second)})}}diff --git a/integration/model_perf_test.go b/integration/model_perf_test.goindex 759e8b9a2..3d6ba9239 100644--- a/integration/model_perf_test.go+++ b/integration/model_perf_test.go@@ -40,6 +40,18 @@ var (// cat int.log | grep MODEL_PERF_HEADER | head -1| cut -f2- -d: > perf.csv// cat int.log | grep MODEL_PERF_DATA | cut -f2- -d: >> perf.csvfunc TestModelsPerf(t *testing.T) {+ if s := os.Getenv("OLLAMA_NEW_ENGINE"); s != "" {+ doModelPerfTest(t, ollamaEngineChatModels)+ } else {+ doModelPerfTest(t, append(ollamaEngineChatModels, llamaRunnerChatModels...))+ }+}++func TestLibraryModelsPerf(t *testing.T) {+ doModelPerfTest(t, libraryChatModels)+}++func doModelPerfTest(t *testing.T, chatModels []string) {softTimeout, hardTimeout := getTimeouts(t)slog.Info("Setting timeouts", "soft", softTimeout, "hard", hardTimeout)ctx, cancel := context.WithTimeout(context.Background(), hardTimeout)@@ -65,14 +77,12 @@ func TestModelsPerf(t *testing.T) {}longPrompt := "summarize the following: " + string(data)- var chatModels []string- if s := os.Getenv("OLLAMA_NEW_ENGINE"); s != "" {- chatModels = ollamaEngineChatModels- } else {- chatModels = append(ollamaEngineChatModels, llamaRunnerChatModels...)- }+ targetArch := os.Getenv("OLLAMA_TEST_ARCHITECTURE")for _, model := range chatModels {+ if !strings.Contains(model, ":") {+ model = model + ":latest"+ }t.Run(model, func(t *testing.T) {if time.Now().Sub(started) > softTimeout {t.Skip("skipping remaining tests to avoid excessive runtime")@@ -88,6 +98,9 @@ func TestModelsPerf(t *testing.T) {}arch := resp.ModelInfo["general.architecture"].(string)maxContext = int(resp.ModelInfo[fmt.Sprintf("%s.context_length", arch)].(float64))+ if targetArch != "" && arch != targetArch {+ t.Skip(fmt.Sprintf("Skipping %s architecture %s != %s", model, arch, targetArch))+ }if maxVram > 0 {resp, err := client.List(ctx)@@ -151,8 +164,8 @@ func TestModelsPerf(t *testing.T) {prompt stringanyResp []string}{- {"why is the sky blue?", []string{"rayleigh", "scattering", "atmosphere", "nitrogen", "oxygen"}},- {maxPrompt, []string{"shakespeare", "oppression", "sorrows", "gutenberg", "child", "license", "sonnet", "melancholy"}},+ {blueSkyPrompt, blueSkyExpected},+ {maxPrompt, []string{"shakespeare", "oppression", "sorrows", "gutenberg", "child", "license", "sonnet", "melancholy", "love", "sorrow", "beauty"}},}var gpuPercent intfor _, tc := range testCases {@@ -241,11 +254,12 @@ func TestModelsPerf(t *testing.T) {}}}+ // Round the logged prompt count for comparisons across versions/configurations which can vary slightlyfmt.Fprintf(os.Stderr, "MODEL_PERF_HEADER:%s,%s,%s,%s,%s,%s,%s\n","MODEL","CONTEXT","GPU PERCENT",- "PROMPT COUNT",+ "APPROX PROMPT COUNT","LOAD TIME","PROMPT EVAL TPS","EVAL TPS",@@ -254,7 +268,7 @@ func TestModelsPerf(t *testing.T) {model,numCtx,gpuPercent,- resp.PromptEvalCount,+ (resp.PromptEvalCount/10)*10,float64(resp.LoadDuration)/1000000000.0,float64(resp.PromptEvalCount)/(float64(resp.PromptEvalDuration)/1000000000.0),float64(resp.EvalCount)/(float64(resp.EvalDuration)/1000000000.0),diff --git a/integration/quantization_test.go b/integration/quantization_test.goindex af9da0b62..305647496 100644--- a/integration/quantization_test.go+++ b/integration/quantization_test.go@@ -76,7 +76,7 @@ func TestQuantization(t *testing.T) {stream := truegenReq := api.GenerateRequest{Model: newName,- Prompt: "why is the sky blue?",+ Prompt: blueSkyPrompt,KeepAlive: &api.Duration{Duration: 3 * time.Second},Options: map[string]any{"seed": 42,@@ -88,14 +88,13 @@ func TestQuantization(t *testing.T) {// Some smaller quantizations can cause models to have poor quality// or get stuck in repetition loops, so we stop as soon as we have any matches- anyResp := []string{"rayleigh", "scattering", "day", "sun", "moon", "color", "nitrogen", "oxygen"}reqCtx, reqCancel := context.WithCancel(ctx)atLeastOne := falsevar buf bytes.Buffergenfn := func(response api.GenerateResponse) error {buf.Write([]byte(response.Response))fullResp := strings.ToLower(buf.String())- for _, resp := range anyResp {+ for _, resp := range blueSkyExpected {if strings.Contains(fullResp, resp) {atLeastOne = truet.Log(fullResp)diff --git a/integration/utils_test.go b/integration/utils_test.goindex 7901fed3f..f8ec13f39 100644--- a/integration/utils_test.go+++ b/integration/utils_test.go@@ -256,13 +256,29 @@ var ("snowflake-arctic-embed","snowflake-arctic-embed2",}++ blueSkyPrompt = "why is the sky blue? Be brief but factual in your reply"+ blueSkyExpected = []string{"rayleigh", "scatter", "atmosphere", "nitrogen", "oxygen", "wavelength", "interact"}++ rainbowPrompt = "how do rainbows form? Be brief but factual in your reply"+ rainbowFollowups = []string{+ "Explain the physics involved in them. Be breif in your reply",+ "Explain the chemistry involved in them. Be breif in your reply",+ "Explain the quantum mechanics involved in them. Be breif in your reply",+ "What are common myths related to them? Be brief in your reply",+ "What are common fairytales related to them? Be brief in your reply",+ "Can they form if there is no rain? Be breif in your reply",+ "Can they form if there are no clouds? Be breif in your reply",+ "Do they happen on other planets? Be brief in your reply",+ }+ rainbowExpected = []string{"water", "droplet", "mist", "glow", "refracted", "reflect", "color", "spectrum", "frequency", "end", "gold", "fortune", "blessing", "prosperity"})func init() {lifecycle.InitLogging()- custom := os.Getenv("OLLAMA_TEST_SMOL_MODEL")+ custom := os.Getenv("OLLAMA_TEST_DEFAULT_MODEL")if custom != "" {- slog.Info("setting smol test model to " + custom)+ slog.Info("setting default test model to " + custom)smol = custom}}@@ -577,11 +593,11 @@ func GenerateRequests() ([]api.GenerateRequest, [][]string) {},},[][]string{- {"sunlight", "scattering", "interact", "color", "surface", "depth", "red", "orange", "yellow", "absorbs", "wavelength"},- {"soil", "organic", "earth", "black", "tan", "chemical", "processes", "pigments", "particles", "iron oxide", "rust", "air", "water", "mixture", "mixing"},- {"water", "droplet", "refracted", "reflect", "color", "spectrum"},+ {"sunlight", "scatter", "interact", "color", "surface", "depth", "red", "orange", "yellow", "absorb", "wavelength", "water", "molecule"},+ {"soil", "organic", "earth", "black", "tan", "chemical", "processes", "pigment", "particle", "iron oxide", "rust", "air", "water", "wet", "mixture", "mixing", "mineral", "element", "decomposed", "matter", "wavelength"},+ {"water", "droplet", "refract", "reflect", "color", "spectrum", "raindrop"},{"fourth", "july", "declaration", "independence"},- {"nitrogen", "oxygen", "carbon", "dioxide", "water", "vapor"},+ {"nitrogen", "oxygen", "carbon", "dioxide", "water", "vapor", "fluid", "particles", "gas"},}}