Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

16.2 KB
Raw Blame
package provider

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"path/filepath"
	"slices"
	"strconv"
	"strings"
	"sync"
	"testing"

	"github.com/yetone/magpie/internal/catalog"
)

// Each endpoint is tried with a key made for it, and the Anthropic one
// with a model its key sees even when the exposed one is another's.
func TestTestUsesEachEndpointsKey(t *testing.T) {
	home := t.TempDir()
	t.Setenv("HOME", home)
	t.Setenv("USERPROFILE", home)
	t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config"))
	t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache"))
	seen := map[string]string{} // path → key and model it got
	srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodGet { // each key sees its own group's models
			if r.Header.Get("Authorization") == "Bearer sk-anthropic" {
				_, _ = rw.Write([]byte(`{"data":[{"id":"claude-opus-5"}]}`))
			} else {
				_, _ = rw.Write([]byte(`{"data":[{"id":"gpt-6-sol"}]}`))
			}
			return
		}
		var in struct{ Model string }
		_ = json.NewDecoder(r.Body).Decode(&in)
		seen[r.URL.Path] = r.Header.Get("Authorization") + " " + in.Model
		_, _ = rw.Write([]byte(`{}`))
	}))
	defer srv.Close()
	p := Provider{ID: "relay", Name: "Relay", Chat: srv.URL + "/v1", Anthropic: srv.URL,
		Key: "sk-chat", KeyProtocol: Chat, Models: []string{"gpt-6-sol"},
		Keys: []KeyAccount{{Key: "sk-anthropic", Protocol: Anthropic}}}
	for _, r := range p.Test(context.Background()) {
		if !r.OK {
			t.Errorf("%s: %+v", r.Protocol, r)
		}
	}
	if got := seen["/v1/chat/completions"]; got != "Bearer sk-chat gpt-6-sol" {
		t.Errorf("chat got %q", got)
	}
	if got := seen["/v1/messages"]; got != "Bearer sk-anthropic claude-opus-5" {
		t.Errorf("anthropic got %q", got)
	}
	ms := p.Available()
	if len(ms) != 2 {
		t.Fatalf("merged list %+v", ms)
	}
	chat, anth := p.first(), p.Keys[0]
	if !p.Serves(chat, "gpt-6-sol") || p.Serves(chat, "claude-opus-5") || !p.Serves(anth, "claude-opus-5") || p.Serves(anth, "gpt-6-sol") {
		t.Error("each key serves only its own list")
	}
	if !p.Serves(anth, "some-unlisted-model") {
		t.Error("a model no list has is anyone's")
	}
}

// Each model is asked on its own, with a key that sees it: one the vendor
// doesn't serve fails while the others answer.
func TestTestModels(t *testing.T) {
	home := t.TempDir()
	t.Setenv("HOME", home)
	t.Setenv("USERPROFILE", home)
	t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config"))
	t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache"))
	var mu sync.Mutex
	seen := map[string]string{} // model → path and key it came with
	srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodGet {
			if r.Header.Get("Authorization") == "Bearer sk-anthropic" {
				_, _ = rw.Write([]byte(`{"data":[{"id":"claude-opus-5"}]}`))
			} else {
				_, _ = rw.Write([]byte(`{"data":[{"id":"gpt-6-sol"},{"id":"gone"}]}`))
			}
			return
		}
		var in struct{ Model string }
		_ = json.NewDecoder(r.Body).Decode(&in)
		mu.Lock()
		seen[in.Model] = r.URL.Path + " " + r.Header.Get("Authorization")
		mu.Unlock()
		if in.Model == "gone" {
			rw.WriteHeader(http.StatusNotFound)
			_, _ = rw.Write([]byte(`{"error":{"message":"no channel for model gone"}}`))
			return
		}
		_, _ = rw.Write([]byte(`{}`))
	}))
	defer srv.Close()
	p := Provider{ID: "relay", Name: "Relay", Chat: srv.URL + "/v1", Anthropic: srv.URL,
		Key: "sk-chat", KeyProtocol: Chat,
		Keys: []KeyAccount{{Key: "sk-anthropic", Protocol: Anthropic}}}
	p.Fetch(context.Background())
	rs := p.TestModels(context.Background(), []string{"gpt-6-sol", "claude-opus-5", "gone"})
	if len(rs) != 3 || !rs[0].OK || !rs[1].OK || rs[2].OK || rs[2].Status != 404 || rs[2].Error != "no channel for model gone" {
		t.Fatalf("results %+v", rs)
	}
	if rs[0].Model != "gpt-6-sol" || rs[1].Protocol != Anthropic || rs[2].Model != "gone" {
		t.Fatalf("order %+v", rs)
	}
	if seen["gpt-6-sol"] != "/v1/chat/completions Bearer sk-chat" || seen["claude-opus-5"] != "/v1/messages Bearer sk-anthropic" {
		t.Fatalf("seen %v", seen)
	}
}

// An image model is tested as the gateway draws with it, on the images
// API with one small picture — a chat request it would turn away — while a
// chat model is still asked in chat. The vendor-wide Test picks the chat one.
func TestTestModelsDraws(t *testing.T) {
	home := t.TempDir()
	t.Setenv("HOME", home)
	t.Setenv("USERPROFILE", home)
	t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config"))
	t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache"))
	var mu sync.Mutex
	seen := map[string]string{} // model → path it was asked on
	bodies := map[string]map[string]any{}
	srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodGet {
			rw.WriteHeader(http.StatusNotFound)
			return
		}
		var in map[string]any
		_ = json.NewDecoder(r.Body).Decode(&in)
		model, _ := in["model"].(string)
		mu.Lock()
		seen[model] = r.URL.Path
		bodies[model] = in
		mu.Unlock()
		draws := strings.Contains(model, "image") || strings.Contains(model, "flux")
		switch {
		case r.URL.Path == "/v1/images/generations" && draws:
			_, _ = rw.Write([]byte(`{"data":[{"url":"https://img.example/1.png"}]}`))
		case r.URL.Path == "/v1/chat/completions" && !draws:
			_, _ = rw.Write([]byte(`{"choices":[{"message":{"content":"hi"}}]}`))
		default:
			rw.WriteHeader(http.StatusBadRequest)
			_, _ = rw.Write([]byte(`{"error":{"message":"wrong endpoint for ` + model + `"}}`))
		}
	}))
	defer srv.Close()
	p := Provider{ID: "custom", Name: "Custom", Chat: srv.URL + "/v1", Key: "sk-x",
		Models: []string{"gpt-image-1", "flux-kontext-pro", "gpt-6-sol"}}
	rs := p.TestModels(context.Background(), []string{"gpt-image-1", "flux-kontext-pro", "gpt-6-sol"})
	for _, r := range rs {
		if !r.OK {
			t.Fatalf("results %+v", rs)
		}
	}
	if seen["gpt-image-1"] != "/v1/images/generations" || seen["flux-kontext-pro"] != "/v1/images/generations" || seen["gpt-6-sol"] != "/v1/chat/completions" {
		t.Fatalf("seen %v", seen)
	}
	if b := bodies["gpt-image-1"]; b["n"] != float64(1) || b["quality"] != "low" || b["prompt"] == "" {
		t.Fatalf("gpt-image body %v", b)
	}
	if b := bodies["flux-kontext-pro"]; b["quality"] != nil || b["messages"] != nil {
		t.Fatalf("flux body %v", b)
	}
	// the vendor-wide Test asks the chat model, though an image one is first
	mu.Lock()
	clear(seen)
	mu.Unlock()
	if rs := p.Test(context.Background()); len(rs) != 1 || !rs[0].OK || rs[0].Model != "gpt-6-sol" {
		t.Fatalf("Test %+v", rs)
	}
}

// #1837: an image model whose name doesn't place it on the images API
// (agnes-image-2.5-flash reads like gemini's -flash image models, which
// chat) is turned away in chat with the vendor naming the images API; the
// gateway draws with it there, so its test passes there too, and a chat
// model the vendor turns away the same way stays failed.
func TestTestModelsGoesWhereTheVendorPointsIt(t *testing.T) {
	home := t.TempDir()
	t.Setenv("HOME", home)
	t.Setenv("USERPROFILE", home)
	t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config"))
	t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache"))
	var mu sync.Mutex
	var paths []string
	srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodGet {
			rw.WriteHeader(http.StatusNotFound)
			return
		}
		var in map[string]any
		_ = json.NewDecoder(r.Body).Decode(&in)
		model, _ := in["model"].(string)
		mu.Lock()
		paths = append(paths, model+" "+r.URL.Path)
		mu.Unlock()
		if r.URL.Path == "/v1/images/generations" && model == "agnes-image-2.5-flash" {
			_, _ = rw.Write([]byte(`{"created":1,"data":[{"url":"https://img.example/1.png"}]}`))
			return
		}
		rw.WriteHeader(http.StatusBadRequest)
		_, _ = rw.Write([]byte(`{"error":{"message":"Model ` + model + ` is an image model. Use /v1/images/generations","type":"invalid_request_error"}}`))
	}))
	defer srv.Close()
	p := Provider{ID: "agnes", Name: "Agnes", Chat: srv.URL + "/v1", Key: "sk-x",
		Models: []string{"agnes-image-2.5-flash", "agnes-chat"}}
	rs := p.TestModels(context.Background(), []string{"agnes-image-2.5-flash", "agnes-chat"})
	if len(rs) != 2 || !rs[0].OK || rs[1].OK {
		t.Fatalf("results %+v", rs)
	}
	mu.Lock()
	defer mu.Unlock()
	slices.Sort(paths)
	want := []string{"agnes-chat /v1/chat/completions", "agnes-image-2.5-flash /v1/chat/completions", "agnes-image-2.5-flash /v1/images/generations"}
	if !slices.Equal(paths, want) {
		t.Fatalf("asked %v, want %v", paths, want)
	}
}

// #3766 (char1eslu): AgentRouter serves gpt-5.6-sol on /v1/responses and
// turns it away on /v1/chat/completions, in Chinese. Test models asked
// chat only, so the model stayed red while the gateway answered it on
// Responses. On that word the test now asks the next API the provider has,
// with that API's key, as the gateway would; another refusal stays as it
// was, and so does the word when no other API is left or every API says
// it, the first API's answer then.
func TestTestModelsMovesToTheAPIServingIt(t *testing.T) {
	oneHome(t)
	said := "模型 gpt-5.6-sol 不支持 /v1/chat/completions 接口,请改用 /v1/responses(OpenAI Responses API)。"
	// the report has the message only: the JSON around it is a guess, and
	// so is how it is encoded (escaped as Python's json.dumps writes it)
	refusal := `{"error":{"message":"` + said + `","type":"invalid_request_error"}}`
	escaped := `{"error":{"message":` + strconv.QuoteToASCII(said) + `,"type":"invalid_request_error"}}`
	var mu sync.Mutex
	var asked []string
	chatSays := refusal
	srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		var in struct{ Model string }
		_ = json.NewDecoder(r.Body).Decode(&in)
		mu.Lock()
		asked = append(asked, in.Model+" "+r.URL.Path+" "+r.Header.Get("Authorization"))
		chat := chatSays
		mu.Unlock()
		switch {
		case in.Model == "gone":
			// new-api's words for a model no channel serves, written from
			// memory, not from the report
			rw.WriteHeader(http.StatusBadRequest)
			_, _ = rw.Write([]byte(`{"error":{"message":"当前分组 default 下对于模型 gone 无可用渠道"}}`))
		case r.URL.Path == "/v1/chat/completions":
			rw.WriteHeader(http.StatusBadRequest)
			_, _ = rw.Write([]byte(chat))
		case in.Model == "nowhere":
			rw.WriteHeader(http.StatusBadRequest)
			_, _ = rw.Write([]byte(`{"error":{"message":"This model is not supported in the v1/responses endpoint."}}`))
		default:
			_, _ = rw.Write([]byte(`{}`))
		}
	}))
	defer srv.Close()
	took := func() []string {
		mu.Lock()
		defer mu.Unlock()
		out := asked
		asked = nil
		return out
	}
	p := Provider{ID: "agentrouter", Name: "AgentRouter", Chat: srv.URL + "/v1", Responses: srv.URL + "/v1",
		Key: "sk-chat", KeyProtocol: Chat, Keys: []KeyAccount{{Key: "sk-responses", Protocol: Responses}}}
	for _, chat := range []string{refusal, escaped} {
		mu.Lock()
		chatSays = chat
		mu.Unlock()
		rs := p.TestModels(context.Background(), []string{"gpt-5.6-sol"})
		if len(rs) != 1 || !rs[0].OK || rs[0].Protocol != Responses {
			t.Fatalf("%s: results %+v", chat, rs)
		}
		want := []string{"gpt-5.6-sol /v1/chat/completions Bearer sk-chat", "gpt-5.6-sol /v1/responses Bearer sk-responses"}
		if got := took(); !slices.Equal(got, want) {
			t.Fatalf("%s: asked %v, want %v", chat, got, want)
		}
	}
	mu.Lock()
	chatSays = refusal
	mu.Unlock()
	// a refusal about the model is its answer, on the first API alone
	rs := p.TestModels(context.Background(), []string{"gone"})
	if len(rs) != 1 || rs[0].OK || rs[0].Protocol != Chat || rs[0].Status != 400 {
		t.Fatalf("gone: results %+v", rs)
	}
	if got := took(); len(got) != 1 {
		t.Fatalf("gone: asked %v, want chat once", got)
	}
	// with chat alone the vendor's word is the answer
	chatOnly := Provider{ID: "agentrouter", Name: "AgentRouter", Chat: srv.URL + "/v1", Key: "sk-chat"}
	rs = chatOnly.TestModels(context.Background(), []string{"gpt-5.6-sol"})
	if len(rs) != 1 || rs[0].OK || rs[0].Protocol != Chat || rs[0].Status != 400 || !strings.Contains(rs[0].Error, "请改用 /v1/responses") {
		t.Fatalf("chat only: results %+v", rs)
	}
	if got := took(); len(got) != 1 {
		t.Fatalf("chat only: asked %v, want chat once", got)
	}
	// every API says it: each is asked, and the first one's answer stays
	rs = p.TestModels(context.Background(), []string{"nowhere"})
	if len(rs) != 1 || rs[0].OK || rs[0].Protocol != Chat || !strings.Contains(rs[0].Error, "请改用 /v1/responses") {
		t.Fatalf("nowhere: results %+v", rs)
	}
	if got := took(); len(got) != 2 {
		t.Fatalf("nowhere: asked %v, want chat and responses", got)
	}
}

// The API the user picked for a model, or the only one the vendor's list
// names, keeps its test there, as usable keeps the gateway's request:
// chat turning it away is the answer, not a reason to ask Responses
// (#3766).
func TestTestModelsKeepsAModelOnItsAPI(t *testing.T) {
	oneHome(t)
	const refusal = `{"error":{"message":"模型 gpt-5.6-sol 不支持 /v1/chat/completions 接口,请改用 /v1/responses(OpenAI Responses API)。","type":"invalid_request_error"}}`
	var mu sync.Mutex
	var asked []string
	srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		mu.Lock()
		asked = append(asked, r.URL.Path)
		mu.Unlock()
		if r.URL.Path == "/v1/chat/completions" {
			rw.WriteHeader(http.StatusBadRequest)
			_, _ = rw.Write([]byte(refusal))
			return
		}
		_, _ = rw.Write([]byte(`{}`))
	}))
	defer srv.Close()
	p := Provider{ID: "agentrouter", Name: "AgentRouter", Key: "k", Chat: srv.URL + "/v1", Responses: srv.URL + "/v1", Models: []string{"picked", "listed"}}
	if err := Save(p); err != nil {
		t.Fatal(err)
	}
	if err := catalog.SaveLive(p.ID, p.Chat, []catalog.Model{{ID: "picked"}, {ID: "listed", APIs: []string{"chat"}}}); err != nil {
		t.Fatal(err)
	}
	if err := SetModelAPI("agentrouter/picked", "chat"); err != nil {
		t.Fatal(err)
	}
	q, err := Find(p.ID)
	if err != nil {
		t.Fatal(err)
	}
	for _, m := range []string{"picked", "listed"} {
		mu.Lock()
		asked = nil
		mu.Unlock()
		rs := q.TestModels(context.Background(), []string{m})
		mu.Lock()
		n := len(asked)
		mu.Unlock()
		if len(rs) != 1 || rs[0].OK || rs[0].Protocol != Chat || n != 1 {
			t.Errorf("%s: results %+v after %d asks, want red on chat after one", m, rs, n)
		}
	}
}

// Asked as Codex, a model the vendor turns away on Responses with its word
// that Chat serves it is asked there with Codex's headers, as the gateway
// sends them on both; asked as Claude Code it stays red on Messages, where
// alone the gateway sends Claude Code's (#3766).
func TestTestModelsAsAClientMovesAsTheGatewayDoes(t *testing.T) {
	oneHome(t)
	SawClient(ClientClaudeCode, http.Header{"User-Agent": {"claude-cli/2.1.0 (external, cli)"}, "X-App": {"cli"}})
	var mu sync.Mutex
	var asked []string
	srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		mu.Lock()
		asked = append(asked, r.URL.Path+" "+r.Header.Get("User-Agent"))
		mu.Unlock()
		switch r.URL.Path {
		case "/v1/responses":
			rw.WriteHeader(http.StatusBadRequest)
			_, _ = rw.Write([]byte(`{"error":{"message":"This model is not supported in the v1/responses endpoint. Use v1/chat/completions."}}`))
		case "/v1/messages":
			rw.WriteHeader(http.StatusBadRequest)
			_, _ = rw.Write([]byte(`{"type":"error","error":{"type":"invalid_request_error","message":"Model gpt-x is not supported for format anthropic"}}`))
		default:
			rw.Header().Set("Content-Type", "text/event-stream")
			_, _ = rw.Write([]byte("data: {}\n\n"))
		}
	}))
	defer srv.Close()
	p := Provider{ID: "relay", Name: "Relay", Key: "k", Chat: srv.URL + "/v1", Responses: srv.URL + "/v1", Anthropic: srv.URL}
	took := func() []string {
		mu.Lock()
		defer mu.Unlock()
		out := asked
		asked = nil
		return out
	}
	rs := p.TestModels(TestAs(context.Background(), ClientCodex), []string{"gpt-x"})
	got := took()
	if len(rs) != 1 || !rs[0].OK || rs[0].Protocol != Chat || len(got) != 2 || !strings.HasPrefix(got[0], "/v1/responses ") ||
		!strings.HasPrefix(got[1], "/v1/chat/completions codex_cli_rs/") {
		t.Errorf("as Codex: results %+v, asked %q", rs, got)
	}
	rs = p.TestModels(TestAs(context.Background(), ClientClaudeCode), []string{"gpt-x"})
	got = took()
	if len(rs) != 1 || rs[0].OK || rs[0].Protocol != Anthropic || len(got) != 1 || got[0] != "/v1/messages claude-cli/2.1.0 (external, cli)" {
		t.Errorf("as Claude Code: results %+v, asked %q", rs, got)
	}
}