Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
package provider
import (
"context"
"net/http"
"net/http/httptest"
"path/filepath"
"slices"
"strings"
"sync/atomic"
"testing"
"github.com/yetone/magpie/internal/agentenv"
"github.com/yetone/magpie/internal/catalog"
)
// A provider whose protocols are served at bases of their own is asked
// each of them, and the lists are merged (#904): a vendor that lists its
// Claude models at its Anthropic base and its GPT models at its Chat base
// kept only the first list that answered, so the models of every other
// endpoint were invisible in the product — in the agents' model lists, in
// the saved preferences, and even behind a model's fixed protocol, which
// setModelAPI refuses for a model the catalog does not list.
func TestFetchUnionsProtocolLists(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
var chatHits, anthropicHits atomic.Int32
chat := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
chatHits.Add(1)
w.Write([]byte(`{"data":[{"id":"gpt-a","display_name":"GPT A",` +
`"supported_reasoning_levels":[{"effort":"high"}],` +
`"modalities":{"input":["text"]}}]}`))
}))
defer chat.Close()
anthropic := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
anthropicHits.Add(1)
w.Write([]byte(`{"data":[{"id":"claude-b","display_name":"Claude B"}]}`))
}))
defer anthropic.Close()
p := Provider{
ID: "dual",
Name: "Dual",
Chat: chat.URL + "/v1",
Anthropic: anthropic.URL + "/v1",
Key: "sk-d",
}
if err := Save(p); err != nil {
t.Fatal(err)
}
q, err := Find("dual")
if err != nil {
t.Fatal(err)
}
ms, err := q.Fetch(context.Background())
if err != nil {
t.Fatalf("fetch: %v", err)
}
got := map[string]catalog.Model{}
for _, m := range ms {
got[m.ID] = m
}
if len(got) != 2 || got["gpt-a"].ID == "" || got["claude-b"].ID == "" {
t.Fatalf("listed %v, want gpt-a and claude-b together", ms)
}
if chatHits.Load() != 1 || anthropicHits.Load() != 1 {
t.Errorf("asked chat %d times, anthropic %d", chatHits.Load(), anthropicHits.Load())
}
// the list a model came from keeps its own fields
if g := got["gpt-a"]; g.Name != "GPT A" || len(g.Efforts) != 1 || g.Efforts[0] != "high" {
t.Errorf("gpt-a: %+v", g)
}
if g := got["claude-b"]; g.Name != "Claude B" {
t.Errorf("claude-b: %+v", g)
}
// the models an agent is offered, and a model's own protocol setting
// (serves, then SetModelAPI), both read the catalog the fetch kept:
// a model only the second endpoint lists has to be in it
avail := map[string]bool{}
for _, m := range q.Available() {
avail[m.ID] = true
}
if !avail["gpt-a"] || !avail["claude-b"] {
t.Errorf("available %v, want gpt-a and claude-b", q.Available())
}
if !q.serves("claude-b") {
t.Error("claude-b is not a model of the provider")
}
// the base kept for routing is the one the first protocol answered at
if l, err := q.fetchOne(context.Background()); err != nil || l.base != chat.URL+"/v1" {
t.Errorf("base %q (%v), want the chat one", l.base, err)
}
}
// The order the protocols are asked in must not decide which models are
// kept: a provider with a Responses base and an Anthropic base and no Chat
// one is asked both, each list once. Codex's own ask of the Responses
// list (codexEntries) is counted apart (#3854).
func TestFetchUnionsProtocolListsWithoutChat(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
var responsesHits, responsesCodexHits, anthropicHits atomic.Int32
responses := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
if r.URL.Query().Has("client_version") {
responsesCodexHits.Add(1)
} else {
responsesHits.Add(1)
}
w.Write([]byte(`{"data":[{"id":"gpt-a"}]}`))
}))
defer responses.Close()
anthropic := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
anthropicHits.Add(1)
w.Write([]byte(`{"data":[{"id":"claude-b"}]}`))
}))
defer anthropic.Close()
if err := Save(Provider{
ID: "dual2",
Name: "Dual 2",
Responses: responses.URL + "/v1",
Anthropic: anthropic.URL + "/v1",
Key: "sk-d",
}); err != nil {
t.Fatal(err)
}
q, err := Find("dual2")
if err != nil {
t.Fatal(err)
}
ms, err := q.Fetch(context.Background())
if err != nil {
t.Fatalf("fetch: %v", err)
}
if len(ms) != 2 || ms[0].ID != "gpt-a" || ms[1].ID != "claude-b" {
t.Fatalf("listed %v, want gpt-a then claude-b", ms)
}
if responsesHits.Load() != 1 || anthropicHits.Load() != 1 {
t.Errorf("asked responses %d times, anthropic %d", responsesHits.Load(), anthropicHits.Load())
}
if responsesCodexHits.Load() > 1 {
t.Errorf("asked responses as Codex asks %d times, want once at most", responsesCodexHits.Load())
}
}
// One model in two lists is one model: its image capability is merged the
// way the keys' lists are (an explicit text-only answer wins), not taken
// from whichever list was read last.
func TestFetchUnionsSharedModel(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
chat := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
w.Write([]byte(`{"data":[{"id":"gpt-a","display_name":"GPT A",` +
`"supported_reasoning_levels":[{"effort":"high"}],` +
`"modalities":{"input":["text"]}}]}`))
}))
defer chat.Close()
anthropic := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
// the same model, as the Anthropic endpoint answers for it
w.Write([]byte(`{"data":[{"id":"gpt-a","display_name":"GPT A",` +
`"supported_reasoning_levels":[{"effort":"high"}],` +
`"modalities":{"input":["text","image"]}}]}`))
}))
defer anthropic.Close()
if err := Save(Provider{
ID: "shared",
Name: "Shared",
Chat: chat.URL + "/v1",
Anthropic: anthropic.URL + "/v1",
Key: "sk-d",
}); err != nil {
t.Fatal(err)
}
q, err := Find("shared")
if err != nil {
t.Fatal(err)
}
ms, err := q.Fetch(context.Background())
if err != nil {
t.Fatalf("fetch: %v", err)
}
if len(ms) != 1 {
t.Fatalf("listed %v, want one model", ms)
}
m := ms[0]
if m.ID != "gpt-a" || m.Name != "GPT A" || len(m.Efforts) != 1 || m.Efforts[0] != "high" {
t.Errorf("fields lost: %+v", m)
}
if m.ImageInput == nil || *m.ImageInput {
t.Errorf("image input %v, want the explicit text-only answer kept", m.ImageInput)
}
if m.Images {
t.Errorf("images on, want off: %+v", m)
}
}
// A provider whose every protocol answers keeps asking only the bases it
// speaks: one protocol means the one request it made before.
func TestFetchAsksOneProtocolOnce(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
var hits atomic.Int32
one := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
hits.Add(1)
w.Write([]byte(`{"data":[{"id":"gpt-a"},{"id":"gpt-b"}]}`))
}))
defer one.Close()
if err := Save(Provider{ID: "one", Name: "One", Chat: one.URL + "/v1", Key: "sk-o"}); err != nil {
t.Fatal(err)
}
q, err := Find("one")
if err != nil {
t.Fatal(err)
}
ms, err := q.Fetch(context.Background())
if err != nil {
t.Fatalf("fetch: %v", err)
}
if len(ms) != 2 || ms[0].ID != "gpt-a" || ms[1].ID != "gpt-b" {
t.Errorf("listed %v", ms)
}
if hits.Load() != 1 {
t.Errorf("asked %d times, want once", hits.Load())
}
}
// Chat and Responses at one base ask it once, as they did when the first
// answer ended the fetch. Codex's own ask of the Responses list
// (codexEntries, with client_version) is a request of its own, once a
// fetch for a provider added by its address, and is counted apart (#3854).
func TestFetchAsksASharedBaseOnce(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
var hits, codexHits atomic.Int32
shared := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
if r.URL.Query().Has("client_version") {
codexHits.Add(1)
} else {
hits.Add(1)
}
w.Write([]byte(`{"data":[{"id":"gpt-a"}]}`))
}))
defer shared.Close()
if err := Save(Provider{
ID: "same",
Name: "Same",
Chat: shared.URL + "/v1",
Responses: shared.URL + "/v1",
Key: "sk-s",
}); err != nil {
t.Fatal(err)
}
q, err := Find("same")
if err != nil {
t.Fatal(err)
}
if ms, err := q.Fetch(context.Background()); err != nil || len(ms) != 1 {
t.Fatalf("%v %v", ms, err)
}
if hits.Load() != 1 {
t.Errorf("asked %d times, want once", hits.Load())
}
if codexHits.Load() > 1 {
t.Errorf("asked as Codex asks %d times, want once at most", codexHits.Load())
}
}
// Where no protocol answers, the error is the one it was: every base asked
// is named in it, with what each said, and the ids can still be typed in.
func TestFetchUnionsFailsWithEveryURL(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
chat := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.NotFound(w, r)
}))
defer chat.Close()
anthropic := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.NotFound(w, r)
}))
defer anthropic.Close()
if err := Save(Provider{
ID: "dead",
Name: "Dead",
Chat: chat.URL + "/v1",
Anthropic: anthropic.URL + "/v1",
Key: "sk-x",
}); err != nil {
t.Fatal(err)
}
q, err := Find("dead")
if err != nil {
t.Fatal(err)
}
ms, err := q.Fetch(context.Background())
if err == nil {
t.Fatalf("listed %v with no endpoint answering", ms)
}
msg := err.Error()
if !strings.Contains(msg, chat.URL+"/v1/models: 404") || !strings.Contains(msg, anthropic.URL) {
t.Errorf("error does not name every base asked: %s", msg)
}
if !strings.Contains(msg, "by hand") {
t.Errorf("error lost the way out: %s", msg)
}
if ms != nil {
t.Errorf("listed %v with no endpoint answering", ms)
}
if _, _, ok := catalog.Live("dead"); ok {
t.Error("a catalog was kept for a provider that answered nothing")
}
}
// A models URL is where the user said its list is: that URL alone is asked,
// once, and no base is.
func TestFetchModelsURLAsksNoBase(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
var baseHits atomic.Int32
base := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
baseHits.Add(1)
w.Write([]byte(`{"data":[{"id":"gpt-a"}]}`))
}))
defer base.Close()
var urlHits atomic.Int32
list := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
urlHits.Add(1)
w.Write([]byte(`{"data":[{"id":"claude-b"}]}`))
}))
defer list.Close()
if err := Save(Provider{
ID: "listed",
Name: "Listed",
Chat: base.URL + "/v1",
Anthropic: base.URL + "/v1",
ModelsURL: list.URL + "/v1/models",
Key: "sk-l",
}); err != nil {
t.Fatal(err)
}
q, err := Find("listed")
if err != nil {
t.Fatal(err)
}
ms, err := q.Fetch(context.Background())
if err != nil {
t.Fatalf("fetch: %v", err)
}
if len(ms) != 1 || ms[0].ID != "claude-b" {
t.Errorf("listed %v, want the models URL's own list", ms)
}
if urlHits.Load() != 1 {
t.Errorf("asked the models URL %d times, want once", urlHits.Load())
}
if baseHits.Load() != 0 {
t.Errorf("asked a base %d times beside the models URL", baseHits.Load())
}
}
// A protocol of a multi-protocol provider that can't be asked now keeps the
// models it listed last time, as a key that can't be asked does: a failed
// read says nothing about what the vendor serves today, and without it a
// picked Claude model is gone from the user's provider for good (#904,
// yetone's review of the union fetch: an Anthropic base answering 502
// deleted its models from the picks and never brought them back).
func TestFetchKeepsTheModelsAFailedProtocolListed(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
chat := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
w.Write([]byte(`{"data":[{"id":"gpt-a"}]}`))
}))
defer chat.Close()
var list atomic.Value
list.Store(`{"data":[{"id":"claude-b"}]}`)
var down atomic.Bool
anthropic := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if down.Load() {
http.Error(w, "upstream is down", http.StatusBadGateway)
return
}
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
w.Write([]byte(list.Load().(string)))
}))
defer anthropic.Close()
if err := Save(Provider{
ID: "failing",
Name: "Failing",
Chat: chat.URL + "/v1",
Anthropic: anthropic.URL + "/v1",
Key: "sk-f",
}); err != nil {
t.Fatal(err)
}
fetch := func() error {
p, err := Find("failing")
if err != nil {
t.Fatal(err)
}
_, err = p.Fetch(context.Background())
return err
}
if err := fetch(); err != nil {
t.Fatal(err)
}
// both models picked from the list both protocols answered with
p, _ := Find("failing")
p.Models = []string{"gpt-a", "claude-b"}
if err := Save(*p); err != nil {
t.Fatal(err)
}
// the Anthropic base is down: the fetch still succeeds on the Chat one,
// and nothing the failed endpoint listed goes missing
down.Store(true)
if err := fetch(); err != nil {
t.Fatalf("fetch with one protocol down: %v", err)
}
p, _ = Find("failing")
if want := []string{"gpt-a", "claude-b"}; !slices.Equal(p.Models, want) {
t.Fatalf("picks after a protocol failed: %v, want %v", p.Models, want)
}
live, _, ok := p.live()
if !ok {
t.Fatal("no list kept after a protocol failed")
}
if ids := idsOf(live); !slices.Equal(ids, []string{"gpt-a", "claude-b"}) {
t.Fatalf("list after a protocol failed: %v, want gpt-a and claude-b", ids)
}
if !p.serves("claude-b") {
t.Error("claude-b is not a model of the provider any more")
}
// and once the endpoint answers again its real list stands: a model it
// has dropped since is dropped here, so keeping is not keeping forever
down.Store(false)
list.Store(`{"data":[{"id":"claude-c"}]}`)
if err := fetch(); err != nil {
t.Fatal(err)
}
p, _ = Find("failing")
if want := []string{"gpt-a"}; !slices.Equal(p.Models, want) {
t.Fatalf("picks after the endpoint answered again: %v, want %v", p.Models, want)
}
live, _, ok = p.live()
if !ok {
t.Fatal("no list kept after the endpoint answered again")
}
if ids := idsOf(live); !slices.Equal(ids, []string{"gpt-a", "claude-c"}) {
t.Fatalf("list after the endpoint answered again: %v, want gpt-a and claude-c", ids)
}
}
// The other way round, which is what stops the fix from keeping every old
// model: a model only the Chat base listed is gone from that base's list,
// and the Anthropic base failing says nothing about it. Keeping every model
// of a failed fetch would resurrect it (yetone's review).
func TestFetchDropsAGoneModelWhileAnotherProtocolFails(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
var list atomic.Value
list.Store(`{"data":[{"id":"gpt-a"},{"id":"gpt-b"}]}`)
chat := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
w.Write([]byte(list.Load().(string)))
}))
defer chat.Close()
var down atomic.Bool
anthropic := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if down.Load() {
http.Error(w, "upstream is down", http.StatusBadGateway)
return
}
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
w.Write([]byte(`{"data":[{"id":"claude-b"}]}`))
}))
defer anthropic.Close()
if err := Save(Provider{
ID: "half",
Name: "Half",
Chat: chat.URL + "/v1",
Anthropic: anthropic.URL + "/v1",
Key: "sk-h",
}); err != nil {
t.Fatal(err)
}
fetch := func() error {
p, err := Find("half")
if err != nil {
t.Fatal(err)
}
_, err = p.Fetch(context.Background())
return err
}
if err := fetch(); err != nil {
t.Fatal(err)
}
p, _ := Find("half")
p.Models = []string{"gpt-a", "gpt-b", "claude-b"}
if err := Save(*p); err != nil {
t.Fatal(err)
}
// the Chat base has dropped gpt-b, and the Anthropic base is down
list.Store(`{"data":[{"id":"gpt-a"}]}`)
down.Store(true)
if err := fetch(); err != nil {
t.Fatalf("fetch with one protocol down: %v", err)
}
p, _ = Find("half")
if want := []string{"gpt-a", "claude-b"}; !slices.Equal(p.Models, want) {
t.Fatalf("picks after the Chat list dropped a model: %v, want %v", p.Models, want)
}
live, _, ok := p.live()
if !ok {
t.Fatal("no list kept after a protocol failed")
}
if ids := idsOf(live); !slices.Equal(ids, []string{"gpt-a", "claude-b"}) {
t.Fatalf("list after the Chat list dropped a model: %v, want gpt-a and claude-b", ids)
}
}
// The same for a Gemini base, which the union merge had left out: a
// protocol whose list is merged but which is kept in no side is a base a
// fetch cannot find when it fails, so the picked Gemini model it listed was
// read as gone and dropped for good — the very fault the Anthropic base
// above was fixed for (yetone's review of #1006, 2026-10-13).
func TestFetchKeepsTheModelsAFailedGeminiBaseListed(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
chat := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
w.Write([]byte(`{"data":[{"id":"gpt-a"}]}`))
}))
defer chat.Close()
var list atomic.Value
list.Store(`{"models":[{"name":"models/claude-b","supportedGenerationMethods":["generateContent"]}]}`)
var down atomic.Bool
gemini := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if down.Load() {
http.Error(w, "upstream is down", http.StatusBadGateway)
return
}
if r.URL.Path != "/v1beta/models" {
http.NotFound(w, r)
return
}
w.Write([]byte(list.Load().(string)))
}))
defer gemini.Close()
if err := Save(Provider{
ID: "gemfailing",
Name: "GemFailing",
Chat: chat.URL + "/v1",
Gemini: gemini.URL + "/v1beta",
Key: "sk-g",
}); err != nil {
t.Fatal(err)
}
fetch := func() error {
p, err := Find("gemfailing")
if err != nil {
t.Fatal(err)
}
_, err = p.Fetch(context.Background())
return err
}
if err := fetch(); err != nil {
t.Fatal(err)
}
// both models picked from the list both bases answered with
p, _ := Find("gemfailing")
p.Models = []string{"gpt-a", "claude-b"}
if err := Save(*p); err != nil {
t.Fatal(err)
}
// the Gemini base is down: the fetch still succeeds on the Chat one,
// and nothing the failed endpoint listed goes missing
down.Store(true)
if err := fetch(); err != nil {
t.Fatalf("fetch with the Gemini base down: %v", err)
}
p, _ = Find("gemfailing")
if want := []string{"gpt-a", "claude-b"}; !slices.Equal(p.Models, want) {
t.Fatalf("picks after the Gemini base failed: %v, want %v", p.Models, want)
}
live, _, ok := p.live()
if !ok {
t.Fatal("no list kept after the Gemini base failed")
}
if ids := idsOf(live); !slices.Equal(ids, []string{"gpt-a", "claude-b"}) {
t.Fatalf("list after the Gemini base failed: %v, want gpt-a and claude-b", ids)
}
if !p.serves("claude-b") {
t.Error("claude-b is not a model of the provider any more")
}
// and once the Gemini base answers again its real list stands: a model
// it has dropped since is dropped here, so keeping is not keeping
// forever
down.Store(false)
list.Store(`{"models":[{"name":"models/claude-c","supportedGenerationMethods":["generateContent"]}]}`)
if err := fetch(); err != nil {
t.Fatal(err)
}
p, _ = Find("gemfailing")
if want := []string{"gpt-a"}; !slices.Equal(p.Models, want) {
t.Fatalf("picks after the Gemini base answered again: %v, want %v", p.Models, want)
}
live, _, ok = p.live()
if !ok {
t.Fatal("no list kept after the Gemini base answered again")
}
if ids := idsOf(live); !slices.Equal(ids, []string{"gpt-a", "claude-c"}) {
t.Fatalf("list after the Gemini base answered again: %v, want gpt-a and claude-c", ids)
}
}
// And the same the other way round for Gemini: a model the Gemini base has
// really dropped is dropped from the picks while the Chat base is down. It
// is what stops the fix above from keeping every old model by never
// dropping anything (yetone's review).
func TestFetchDropsAGoneGeminiModelWhileAnotherProtocolFails(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
var list atomic.Value
list.Store(`{"data":[{"id":"gpt-a"},{"id":"gpt-b"}]}`)
var down atomic.Bool
chat := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if down.Load() {
http.Error(w, "upstream is down", http.StatusBadGateway)
return
}
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
w.Write([]byte(list.Load().(string)))
}))
defer chat.Close()
var gemList atomic.Value
gemList.Store(`{"models":[{"name":"models/claude-b","supportedGenerationMethods":["generateContent"]},` +
`{"name":"models/claude-c","supportedGenerationMethods":["generateContent"]}]}`)
gemini := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1beta/models" {
http.NotFound(w, r)
return
}
w.Write([]byte(gemList.Load().(string)))
}))
defer gemini.Close()
if err := Save(Provider{
ID: "gemhalf",
Name: "GemHalf",
Chat: chat.URL + "/v1",
Gemini: gemini.URL + "/v1beta",
Key: "sk-h",
}); err != nil {
t.Fatal(err)
}
fetch := func() error {
p, err := Find("gemhalf")
if err != nil {
t.Fatal(err)
}
_, err = p.Fetch(context.Background())
return err
}
if err := fetch(); err != nil {
t.Fatal(err)
}
p, _ := Find("gemhalf")
p.Models = []string{"gpt-a", "gpt-b", "claude-b", "claude-c"}
if err := Save(*p); err != nil {
t.Fatal(err)
}
// the Gemini base has dropped claude-c, and the Chat base is down
gemList.Store(`{"models":[{"name":"models/claude-b","supportedGenerationMethods":["generateContent"]}]}`)
down.Store(true)
if err := fetch(); err != nil {
t.Fatalf("fetch with the Chat base down: %v", err)
}
p, _ = Find("gemhalf")
if want := []string{"gpt-a", "gpt-b", "claude-b"}; !slices.Equal(p.Models, want) {
t.Fatalf("picks after the Gemini list dropped a model: %v, want %v", p.Models, want)
}
live, _, ok := p.live()
if !ok {
t.Fatal("no list kept after a protocol failed")
}
// the bases are asked in order, and what a failed base keeps is added
// beside the list, not in the place it would have answered at
if ids := idsOf(live); !slices.Equal(ids, []string{"claude-b", "gpt-a", "gpt-b"}) {
t.Fatalf("list after the Gemini list dropped a model: %v, want claude-b, gpt-a and gpt-b", ids)
}
}
// idsOf are the ids of a list, in its order.
func idsOf(ms []catalog.Model) []string {
var out []string
for _, m := range ms {
out = append(out, m.ID)
}
return out
}
// The same where each of several keys has a list of its own: a base of the
// provider's that can't be asked keeps its models, and every model's keys
// are its own still.
func TestFetchKeepsTheModelsAFailedProtocolListedPerKey(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
chat := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
w.Write([]byte(`{"data":[{"id":"gpt-a"}]}`))
}))
defer chat.Close()
var down atomic.Bool
anthropic := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if down.Load() {
http.Error(w, "upstream is down", http.StatusBadGateway)
return
}
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
w.Write([]byte(`{"data":[{"id":"claude-b"}]}`))
}))
defer anthropic.Close()
if err := Save(Provider{
ID: "keys",
Name: "Keys",
Chat: chat.URL + "/v1",
Anthropic: anthropic.URL + "/v1",
Key: "sk-1",
Keys: []KeyAccount{{Key: "sk-2"}},
}); err != nil {
t.Fatal(err)
}
fetch := func() error {
p, err := Find("keys")
if err != nil {
t.Fatal(err)
}
_, err = p.Fetch(context.Background())
return err
}
if err := fetch(); err != nil {
t.Fatal(err)
}
p, _ := Find("keys")
p.Models = []string{"gpt-a", "claude-b"}
if err := Save(*p); err != nil {
t.Fatal(err)
}
down.Store(true)
if err := fetch(); err != nil {
t.Fatalf("fetch with one protocol down: %v", err)
}
p, _ = Find("keys")
if want := []string{"gpt-a", "claude-b"}; !slices.Equal(p.Models, want) {
t.Fatalf("picks after a protocol failed: %v, want %v", p.Models, want)
}
live, _, ok := p.live()
if !ok {
t.Fatal("no list kept after a protocol failed")
}
if ids := idsOf(live); !slices.Equal(ids, []string{"gpt-a", "claude-b"}) {
t.Fatalf("list after a protocol failed: %v, want gpt-a and claude-b", ids)
}
// both keys still see both models: a model kept for a base that failed
// is not a model of the keys that answered beside it alone
for _, m := range live {
if len(m.Keys) != 2 || !slices.Contains(m.Keys, KeyID("sk-1")) || !slices.Contains(m.Keys, KeyID("sk-2")) {
t.Errorf("%s keys %v, want both", m.ID, m.Keys)
}
}
}
// A list saved before the parts were kept says nothing about which base
// listed what, so the models no answering base lists are kept this once, and
// the next fetch with every base answering settles it: what that base does
// not list is gone, however long it was kept for.
func TestFetchKeepsModelsOfAListSavedWithoutParts(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
chat := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
w.Write([]byte(`{"data":[{"id":"gpt-a"}]}`))
}))
defer chat.Close()
var down atomic.Bool
anthropic := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if down.Load() {
http.Error(w, "upstream is down", http.StatusBadGateway)
return
}
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
w.Write([]byte(`{"data":[{"id":"claude-b"}]}`))
}))
defer anthropic.Close()
if err := Save(Provider{
ID: "unrecorded",
Name: "Unrecorded",
Chat: chat.URL + "/v1",
Anthropic: anthropic.URL + "/v1",
Key: "sk-u",
}); err != nil {
t.Fatal(err)
}
// the list a fetch before this change kept: the union, with no part
// saying which base listed what
if err := catalog.SaveLive("unrecorded", chat.URL+"/v1", []catalog.Model{
{ID: "gpt-a"}, {ID: "gpt-b"}, {ID: "claude-b"},
}); err != nil {
t.Fatal(err)
}
p, _ := Find("unrecorded")
p.Models = []string{"gpt-a", "gpt-b", "claude-b"}
if err := Save(*p); err != nil {
t.Fatal(err)
}
down.Store(true)
if _, err := p.Fetch(context.Background()); err != nil {
t.Fatalf("fetch with one protocol down: %v", err)
}
p, _ = Find("unrecorded")
if want := []string{"gpt-a", "gpt-b", "claude-b"}; !slices.Equal(p.Models, want) {
t.Fatalf("picks after a protocol failed: %v, want %v", p.Models, want)
}
// every base answering now says which models are its own: gpt-b, gone
// from the Chat list, is gone
down.Store(false)
if _, err := p.Fetch(context.Background()); err != nil {
t.Fatal(err)
}
p, _ = Find("unrecorded")
if want := []string{"gpt-a", "claude-b"}; !slices.Equal(p.Models, want) {
t.Fatalf("picks after every base answered: %v, want %v", p.Models, want)
}
}
// An endpoint that has never answered keeps nothing. The parts saved with
// the list say which endpoint listed what, and an endpoint in none of them
// is one that never answered: the whole list saved before is not its own,
// and a model the endpoint that does answer has dropped is gone, however
// many rounds the other one stays at 404. Reading an absent part as an
// empty one instead fixed that guess as what the endpoint listed, and the
// model could never be dropped again (#904, yetone's review of #1006).
func TestFetchDropsAModelWhileAnEndpointHasNeverAnswered(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
var list atomic.Value
list.Store(`{"data":[{"id":"gpt-a"},{"id":"gpt-b"}]}`)
chat := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
w.Write([]byte(list.Load().(string)))
}))
defer chat.Close()
// the Anthropic base never answers, from the first round on: no part
// of the list is ever its own
anthropic := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.NotFound(w, r)
}))
defer anthropic.Close()
if err := Save(Provider{
ID: "never",
Name: "Never",
Chat: chat.URL + "/v1",
Anthropic: anthropic.URL + "/v1",
Key: "sk-n",
}); err != nil {
t.Fatal(err)
}
fetch := func() error {
p, err := Find("never")
if err != nil {
t.Fatal(err)
}
_, err = p.Fetch(context.Background())
return err
}
// round 0: both models listed by the one endpoint that answers, and
// both picked
if err := fetch(); err != nil {
t.Fatal(err)
}
p, _ := Find("never")
p.Models = []string{"gpt-a", "gpt-b"}
if err := Save(*p); err != nil {
t.Fatal(err)
}
live, _, ok := p.live()
if !ok {
t.Fatal("no list kept after the first round")
}
if ids, want := idsOf(live), []string{"gpt-a", "gpt-b"}; !slices.Equal(ids, want) {
t.Fatalf("list after the first round: %v", ids)
}
// the Chat base has dropped gpt-b. The Anthropic base, which never
// answered, has no say in it: it is gone from the second round on, and
// stays gone however long that base keeps answering 404.
list.Store(`{"data":[{"id":"gpt-a"}]}`)
for round := 1; round <= 4; round++ {
if err := fetch(); err != nil {
t.Fatalf("round %d: %v", round, err)
}
p, _ = Find("never")
if want := []string{"gpt-a"}; !slices.Equal(p.Models, want) {
t.Fatalf("round %d picks: %v, want %v", round, p.Models, want)
}
live, _, ok = p.live()
if !ok {
t.Fatalf("round %d: no list kept", round)
}
if ids, want := idsOf(live), []string{"gpt-a"}; !slices.Equal(ids, want) {
t.Fatalf("round %d list: %v, want %v", round, ids, want)
}
// an endpoint that never answered is not saved as one that listed
// nothing: it has said nothing at all, and a part here would be a
// guess fixed as what it listed
sides, _, ok := catalog.LiveSplit("never")
if !ok {
t.Fatalf("round %d: no parts kept", round)
}
// the Anthropic base is kept as the root /v1/messages is asked at
// (normalize), which is the key its part is saved under
if _, saved := sides[anthropic.URL]; saved {
t.Fatalf("round %d: a part was saved for an endpoint that never answered", round)
}
}
}
// A key that can't be asked at any of its bases keeps the models it saw
// last time, and they keep the bases that listed them: the next time that
// key answers at its Chat base while its Anthropic base is still down, the
// Claude model only that key saw is that base's own, and is kept as a
// single key's failed base is. Kept as a whole list with no part saying
// where it came from it belongs to no base at all, and is read as gone the
// first time that key answers half-way (#904, yetone's review of #1006).
func TestFetchKeepsAModelsBaseAfterItsKeyFailedWhole(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
// each key has a Claude model of its own, so claude-b is one base's own
// and of the second key alone
var chatDown, anthropicDown, swapped atomic.Bool
chat := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
if chatDown.Load() && r.Header.Get("Authorization") == "Bearer sk-2" {
http.NotFound(w, r)
return
}
w.Write([]byte(`{"data":[{"id":"gpt-a"}]}`))
}))
defer chat.Close()
anthropic := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
if r.Header.Get("Authorization") != "Bearer sk-2" {
w.Write([]byte(`{"data":[{"id":"claude-1"}]}`))
return
}
if anthropicDown.Load() {
http.NotFound(w, r)
return
}
if swapped.Load() {
w.Write([]byte(`{"data":[{"id":"claude-c"}]}`))
return
}
w.Write([]byte(`{"data":[{"id":"claude-b"}]}`))
}))
defer anthropic.Close()
if err := Save(Provider{
ID: "wholekey",
Name: "Whole key",
Chat: chat.URL + "/v1",
Anthropic: anthropic.URL + "/v1",
Key: "sk-1",
Keys: []KeyAccount{{Key: "sk-2"}},
}); err != nil {
t.Fatal(err)
}
fetch := func() ([]string, []string) {
p, err := Find("wholekey")
if err != nil {
t.Fatal(err)
}
if _, err := p.Fetch(context.Background()); err != nil {
t.Fatal(err)
}
p, _ = Find("wholekey")
live, _, ok := p.live()
if !ok {
t.Fatal("no list kept")
}
return idsOf(live), p.Models
}
all := []string{"gpt-a", "claude-1", "claude-b"}
// round 0: every base of every key answers
got, _ := fetch()
if !slices.Equal(got, all) {
t.Fatalf("list after the first round: %v, want %v", got, all)
}
p, _ := Find("wholekey")
p.Models = []string{"gpt-a", "claude-b"}
if err := Save(*p); err != nil {
t.Fatal(err)
}
// round 1: the second key can't be asked at any of its bases, so its
// Claude model is kept from the list saved before
chatDown.Store(true)
anthropicDown.Store(true)
got, _ = fetch()
if !slices.Equal(got, all) {
t.Fatalf("list after the second key failed whole: %v, want %v", got, all)
}
// round 2: that key answers at its Chat base again while its Anthropic
// base is still down
chatDown.Store(false)
got, picks := fetch()
if !slices.Equal(got, all) {
t.Fatalf("list after the second key answered half-way: %v, want %v", got, all)
}
if !slices.Equal(picks, []string{"gpt-a", "claude-b"}) {
t.Fatalf("picks after the second key answered half-way: %v, want gpt-a and claude-b", picks)
}
// round 3: that key's Anthropic base answers again and has dropped the
// model, which is then gone: keeping it is not keeping it for good
anthropicDown.Store(false)
swapped.Store(true)
got, picks = fetch()
if want := []string{"gpt-a", "claude-1", "claude-c"}; !slices.Equal(got, want) {
t.Fatalf("list after that base answered again: %v, want %v", got, want)
}
if !slices.Equal(picks, []string{"gpt-a"}) {
t.Fatalf("picks after that base answered again: %v, want gpt-a", picks)
}
}
// The other way round for the same rule, which is what keeps sides saying
// what the bases listed: a model kept for a key that could not be asked at
// all belongs to the bases that listed it, but not to one that answered
// this round for another of the model's keys and no longer serves it. A
// part here said the Anthropic base still served claude-b, and the fetch
// after that read it as a base that couldn't be asked, holding a model its
// vendor had dropped for one round longer (yetone's review of #1006, found
// on the path TestFetchKeepsAModelsBaseAfterItsKeyFailedWhole fixed).
//
// It is the same two keys as that test, and the difference is in one thing:
// there claude-b is only the second key's, so the Anthropic base never
// listed it for the first one and its answering says nothing about it. Here
// the first key saw claude-b too, and this round stopped being served it.
func TestFetchDropsAModelABaseStoppedServingWhileAKeyIsDown(t *testing.T) {
isolate(t)
h := t.TempDir()
t.Setenv("HOME", h)
t.Setenv("USERPROFILE", h)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config"))
t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache"))
t.Setenv("PATH", h)
for _, v := range agentenv.Vars {
t.Setenv(v, "")
}
var secondDown, dropped, anthDown atomic.Bool
chat := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
if secondDown.Load() && r.Header.Get("Authorization") == "Bearer sk-2" {
http.NotFound(w, r)
return
}
w.Write([]byte(`{"data":[{"id":"gpt-a"}]}`))
}))
defer chat.Close()
// both keys are served claude-b to begin with; the drop below is the
// vendor stopping both of them
anthropic := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
if anthDown.Load() {
http.NotFound(w, r)
return
}
if r.Header.Get("Authorization") == "Bearer sk-2" && secondDown.Load() {
http.NotFound(w, r)
return
}
if dropped.Load() {
w.Write([]byte(`{"data":[{"id":"claude-1"}]}`))
return
}
w.Write([]byte(`{"data":[{"id":"claude-1"},{"id":"claude-b"}]}`))
}))
defer anthropic.Close()
if err := Save(Provider{
ID: "stopped",
Name: "Stopped",
Chat: chat.URL + "/v1",
Anthropic: anthropic.URL + "/v1",
Key: "sk-1",
Keys: []KeyAccount{{Key: "sk-2"}},
}); err != nil {
t.Fatal(err)
}
fetch := func() ([]string, []string) {
p, err := Find("stopped")
if err != nil {
t.Fatal(err)
}
if _, err := p.Fetch(context.Background()); err != nil {
t.Fatal(err)
}
p, _ = Find("stopped")
live, _, ok := p.live()
if !ok {
t.Fatal("no list kept")
}
return idsOf(live), p.Models
}
// round 0: both keys see claude-b
got, _ := fetch()
if want := []string{"gpt-a", "claude-1", "claude-b"}; !slices.Equal(got, want) {
t.Fatalf("list after the first round: %v, want %v", got, want)
}
p, _ := Find("stopped")
p.Models = []string{"gpt-a", "claude-b"}
if err := Save(*p); err != nil {
t.Fatal(err)
}
// round 1: the second key is down at both of its bases, so claude-b is
// kept for it, and in the same round the Anthropic base answers for the
// first key without listing it. The base is not that model's any more,
// and no part says it is.
secondDown.Store(true)
dropped.Store(true)
got, _ = fetch()
if want := []string{"gpt-a", "claude-1", "claude-b"}; !slices.Equal(got, want) {
t.Fatalf("list while a key is down and the base dropped a model: %v, want %v", got, want)
}
sides, _, ok := catalog.LiveSplit("stopped")
if !ok {
t.Fatal("no parts kept")
}
if ids := idsOf(sides[anthropic.URL]); !slices.Equal(ids, []string{"claude-1"}) {
t.Fatalf("part of the Anthropic base: %v, want claude-1 alone", ids)
}
// round 2: that key is asked again and its Anthropic base is down for
// both keys, so nothing keeps claude-b and it is gone
secondDown.Store(false)
anthDown.Store(true)
got, picks := fetch()
if want := []string{"gpt-a", "claude-1"}; !slices.Equal(got, want) {
t.Fatalf("list after the base went down for every key: %v, want %v", got, want)
}
if !slices.Equal(picks, []string{"gpt-a"}) {
t.Fatalf("picks after the base went down for every key: %v, want gpt-a", picks)
}
// round 3: and it stays gone once the base answers again without it
anthDown.Store(false)
got, picks = fetch()
if want := []string{"gpt-a", "claude-1"}; !slices.Equal(got, want) {
t.Fatalf("list after the base answered again: %v, want %v", got, want)
}
if !slices.Equal(picks, []string{"gpt-a"}) {
t.Fatalf("picks after the base answered again: %v, want gpt-a", picks)
}
}