Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: Codex's model list is asked of the backend once for every app-server, and kept, so it comes within Codex's 5 s (#3734)
yetone479de8900182
diff --git a/internal/gateway/codex_backend.go b/internal/gateway/codex_backend.goindex 736058a04..6bb026739 100644--- a/internal/gateway/codex_backend.go+++ b/internal/gateway/codex_backend.go@@ -14,6 +14,7 @@ import ("regexp""slices""strings"+ "sync""time""github.com/tidwall/gjson"@@ -1744,40 +1745,162 @@ func copyHeaders(dst, src http.Header) {// them, held magpie's answer past that (#539). A var so tests can say.var codexModelsWait = 3 * time.Second+// codexListFor is how long the backend's list is handed out again without+// asking it; past that it is still handed out at once while it is asked+// again (#3734). A var so tests can say.+var codexListFor = time.Minute++// codexListFetch bounds one ask of the backend, which goes on past+// codexModelsWait for the callers after.+const codexListFetch = 30 * time.Second++// codexLists is the backend's last list for each sign-in. Twenty Codex+// app-servers asked it at once, each on its own and each given up at+// codexModelsWait, so a backend slower than that never got a list through+// and Codex fell back to its own metadata, without tool_search for+// magpie's groups, and inlined every Apps tool: ~500K tokens a turn+// (#3734, 1,906 timeouts in a day).+var codexLists = struct {+ sync.Mutex+ at map[string]*codexListed+ // asking is the asks going, for tests to wait on+ asking sync.WaitGroup+}{at: map[string]*codexListed{}}++type codexListed struct {+ body []byte // the list as the backend said it, nil for none yet+ etag string+ got time.Time+ // deny is a 401 or 403 the last ask got, Codex's to see+ deny *http.Response+ denyBody []byte+ // asked is closed when the ask going ends; nil when none is+ asked chan struct{}+}++// codexListKey is whose list it is: the backend's, the account's (a+// token renewed is the same account), the version Codex says.+func codexListKey(r *http.Request) string {+ who := r.Header.Get("chatgpt-account-id")+ if who == "" {+ who = r.Header.Get("Authorization")+ }+ return provider.CodexBase + "\x00" + who + "\x00" + r.URL.RawQuery+}++// codexBackendList is the backend's model list for r's sign-in: the last+// one asked when it is fresh, or at once while it is asked again; else+// what an ask brings within codexModelsWait. One ask at a time for each+// sign-in, whoever is waiting on it. body is nil when there is none;+// deny, when the backend refused the sign-in.+func (s *Server) codexBackendList(r *http.Request) (body []byte, etag string, deny *http.Response, denyBody []byte) {+ key := codexListKey(r)+ codexLists.Lock()+ e := codexLists.at[key]+ if e == nil {+ e = &codexListed{}+ codexLists.at[key] = e+ }+ if e.body != nil && time.Since(e.got) < codexListFor {+ defer codexLists.Unlock()+ return e.body, e.etag, nil, nil+ }+ if e.asked == nil {+ // a refusal an ask before had is no answer for this one+ e.asked, e.deny, e.denyBody = make(chan struct{}), nil, nil+ codexLists.asking.Add(1)+ go s.askCodexList(r.Context(), r.Header.Clone(), r.URL.RawQuery, e, e.asked)+ }+ asked, stale := e.asked, e.body != nil+ codexLists.Unlock()+ if !stale {+ t := time.NewTimer(codexModelsWait)+ select {+ case <-asked:+ case <-t.C:+ case <-r.Context().Done():+ }+ t.Stop()+ }+ codexLists.Lock()+ defer codexLists.Unlock()+ return e.body, e.etag, e.deny, e.denyBody+}++// askCodexList asks the backend for its list into e, past the request+// that started it, and closes done.+func (s *Server) askCodexList(rctx context.Context, h http.Header, query string, e *codexListed, done chan struct{}) {+ defer codexLists.asking.Done()+ var body, denyBody []byte+ var etag string+ var deny *http.Response+ defer func() {+ codexLists.Lock()+ if body != nil {+ e.body, e.etag, e.got, e.deny, e.denyBody = body, etag, time.Now(), nil, nil+ } else if deny != nil {+ // a sign-in to renew: the next caller waits on the backend+ // again and sees it, nothing stale handed out meanwhile+ e.body, e.deny, e.denyBody = nil, deny, denyBody+ }+ e.asked = nil+ codexLists.Unlock()+ close(done)+ }()+ u := provider.CodexBase + "/models"+ if query != "" {+ u += "?" + query+ }+ ctx, cancel := context.WithTimeout(provider.ViaSignedIn(context.WithoutCancel(rctx), "codex"), codexListFetch)+ defer cancel()+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)+ if err != nil {+ return+ }+ copyHeaders(req.Header, h)+ // the whole list, shared by every caller, whatever one of them had+ for _, k := range []string{"Accept-Encoding", "If-None-Match", "If-Modified-Since"} {+ req.Header.Del(k)+ }+ res, err := s.client.Do(req)+ if err != nil {+ return+ }+ b, _ := io.ReadAll(io.LimitReader(res.Body, 16<<20))+ res.Body.Close()+ var list struct {+ Models []any `json:"models"`+ }+ if res.StatusCode < 300 && json.Unmarshal(b, &list) == nil && list.Models != nil {+ body, etag = b, res.Header.Get("ETag")+ // the versions as the backend says them, before any is stamped+ // (see codexcat.V1)+ codexcat.Remember(list.Models)+ } else if res.StatusCode == 401 || res.StatusCode == 403 {+ deny, denyBody = &http.Response{StatusCode: res.StatusCode, Header: res.Header}, b+ }+}+// codexModels is the ChatGPT backend's model list for this sign-in, with// magpie's models after it. Should the backend not answer, or not in// time, Codex's last list of its own stands in.func (s *Server) codexModels(w http.ResponseWriter, r *http.Request) {var own []any- etag := ""- u := provider.CodexBase + "/models"- if r.URL.RawQuery != "" {- u += "?" + r.URL.RawQuery+ body, etag, deny, denyBody := s.codexBackendList(r)+ if deny != nil {+ // a sign-in to renew is Codex's to see+ w.Header().Set("Content-Type", deny.Header.Get("Content-Type"))+ w.WriteHeader(deny.StatusCode)+ w.Write(denyBody)+ return}- ctx, cancel := context.WithTimeout(provider.ViaSignedIn(r.Context(), "codex"), codexModelsWait)- defer cancel()- if req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil); err == nil {- copyHeaders(req.Header, r.Header)- req.Header.Del("Accept-Encoding")- if res, err := s.client.Do(req); err == nil {- var list struct {- Models []any `json:"models"`- }- b, _ := io.ReadAll(io.LimitReader(res.Body, 16<<20))- res.Body.Close()- if res.StatusCode < 300 && json.Unmarshal(b, &list) == nil {- own, etag = list.Models, res.Header.Get("ETag")- // the versions as the backend says them, before any is- // stamped below (see codexcat.V1)- codexcat.Remember(own)- } else if res.StatusCode == 401 || res.StatusCode == 403 {- // a sign-in to renew is Codex's to see- w.Header().Set("Content-Type", res.Header.Get("Content-Type"))- w.WriteHeader(res.StatusCode)- w.Write(b)- return- }+ if body != nil {+ // each caller's own copy: what follows writes into the entries+ var list struct {+ Models []any `json:"models"`}+ json.Unmarshal(body, &list)+ own = list.Models}// what follows reads the providers seven times over (the catalog, the// codex provider's picks and windows, the list's tag): held, they arediff --git a/internal/gateway/codex_backend_test.go b/internal/gateway/codex_backend_test.goindex f7b246043..7415d04d3 100644--- a/internal/gateway/codex_backend_test.go+++ b/internal/gateway/codex_backend_test.go@@ -24,6 +24,7 @@ import (func chatgpt(t *testing.T, h http.HandlerFunc) *httptest.Server {t.Helper()setHome(t, t.TempDir()) // no sign-ins but the test's+ forgetCodexLists(t)up := httptest.NewServer(h)t.Cleanup(up.Close)was := provider.CodexBasediff --git a/internal/gateway/codex_models_shared_test.go b/internal/gateway/codex_models_shared_test.gonew file mode 100644index 000000000..0dfe0e118--- /dev/null+++ b/internal/gateway/codex_models_shared_test.go@@ -0,0 +1,235 @@+package gateway++import (+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/yetone/magpie/internal/provider"+)++// forgetCodexLists starts the test with no backend list kept, and ends it+// with every ask of the backend it started done.+func forgetCodexLists(t *testing.T) {+ t.Helper()+ reset := func() {+ codexLists.asking.Wait()+ codexLists.Lock()+ codexLists.at = map[string]*codexListed{}+ codexLists.Unlock()+ }+ reset()+ t.Cleanup(reset)+}++// codexListAsk is one /models request as a Codex app-server sends it.+func codexListAsk(h http.Handler) (int, map[string]bool) {+ rec := httptest.NewRecorder()+ req := httptest.NewRequest("GET", CodexPath+"/models?client_version=0.164.0", nil)+ req.Header.Set("Authorization", "Bearer chatgpt-token")+ req.Header.Set("chatgpt-account-id", "acct-1")+ h.ServeHTTP(rec, req)+ var list struct {+ Models []map[string]any `json:"models"`+ }+ json.Unmarshal(rec.Body.Bytes(), &list)+ has := map[string]bool{}+ for _, m := range list.Models {+ s, _ := m["slug"].(string)+ has[s] = true+ }+ return rec.Code, has+}++// Twenty Codex app-servers asking for the model list at once, with a+// ChatGPT backend slower than magpie waits for it (#3734): the backend is+// asked once for all of them, and its list, though it came after they+// were answered, is what the next ones get, at once, with magpie's+// models. Before, each asked on its own and gave up at codexModelsWait,+// so a slow backend's list never came through.+func TestCodexModelListSharedWhenSlow(t *testing.T) {+ setup(t, provider.Chat, &fake{t: t})+ var asks atomic.Int32+ hold := make(chan struct{})+ var holdOnce sync.Once+ letGo := func() { holdOnce.Do(func() { close(hold) }) }+ chatgpt(t, func(w http.ResponseWriter, r *http.Request) {+ asks.Add(1)+ select {+ case <-hold:+ case <-r.Context().Done():+ return+ }+ w.Write([]byte(`{"models":[{"slug":"gpt-live","priority":1}]}`))+ })+ t.Cleanup(letGo)+ home, _ := os.UserHomeDir()+ os.MkdirAll(filepath.Join(home, ".codex"), 0o755)+ os.WriteFile(filepath.Join(home, ".codex", "models_cache.json"),+ []byte(`{"etag":"\"v1\"","models":[{"slug":"gpt-cached","priority":1,"base_instructions":"x"}]}`), 0o644)+ was := codexModelsWait+ codexModelsWait = 200 * time.Millisecond+ t.Cleanup(func() { codexModelsWait = was })+ h := New().Handler()++ // the backend held: every one answered from Codex's last list, the+ // backend asked once+ var wg sync.WaitGroup+ var bad atomic.Int32+ for range 20 {+ wg.Add(1)+ go func() {+ defer wg.Done()+ if code, has := codexListAsk(h); code != 200 || !has["fake/m1"] || !has["gpt-cached"] {+ bad.Add(1)+ }+ }()+ }+ wg.Wait()+ if bad.Load() != 0 {+ t.Fatalf("%d of 20 answers without magpie's models or Codex's last list", bad.Load())+ }+ if n := asks.Load(); n != 1 {+ t.Fatalf("the backend was asked %d times for 20 lists at once, want 1", n)+ }++ // it answers late; the ask was not given up with the first answers+ letGo()+ deadline := time.Now().Add(10 * time.Second)+ for {+ _, has := codexListAsk(h)+ if has["gpt-live"] {+ if !has["fake/m1"] || has["gpt-cached"] {+ t.Fatalf("the backend's list without magpie's models, or beside the cached one: %v", has)+ }+ break+ }+ if time.Now().After(deadline) {+ t.Fatal("the backend's list, answered after magpie stopped waiting, never reached a caller")+ }+ time.Sleep(20 * time.Millisecond)+ }++ // fresh, it is handed out again without asking+ n := asks.Load()+ for range 5 {+ if _, has := codexListAsk(h); !has["gpt-live"] || !has["fake/m1"] {+ t.Fatalf("a fresh list not handed out again: %v", has)+ }+ }+ if got := asks.Load() - n; got != 0 {+ t.Fatalf("the backend was asked %d times more within codexListFor", got)+ }+}++// Past codexListFor the last list is handed out at once while the backend+// is asked again, so a slow backend holds no answer once it has answered+// one; what it says then is what the callers after get.+func TestCodexModelListStaleAnsweredAtOnce(t *testing.T) {+ setup(t, provider.Chat, &fake{t: t})+ var asks atomic.Int32+ slow := make(chan struct{})+ chatgpt(t, func(w http.ResponseWriter, r *http.Request) {+ if asks.Add(1) == 1 {+ w.Write([]byte(`{"models":[{"slug":"gpt-old","priority":1}]}`))+ return+ }+ select {+ case <-slow:+ case <-r.Context().Done():+ return+ }+ w.Write([]byte(`{"models":[{"slug":"gpt-new","priority":1}]}`))+ })+ t.Cleanup(func() { close(slow) })+ was, wasFor := codexModelsWait, codexListFor+ codexModelsWait, codexListFor = 3*time.Second, time.Nanosecond+ t.Cleanup(func() { codexModelsWait, codexListFor = was, wasFor })+ h := New().Handler()+ if _, has := codexListAsk(h); !has["gpt-old"] {+ t.Fatalf("first list %v", has)+ }+ // stale at once; the backend now holds its answer. Each answer comes+ // from the list it has, well inside the 3 s it would have waited+ start := time.Now()+ for range 3 {+ if _, has := codexListAsk(h); !has["gpt-old"] || !has["fake/m1"] {+ t.Fatalf("a stale list not handed out: %v", has)+ }+ }+ if d := time.Since(start); d >= codexModelsWait {+ t.Fatalf("three answers took %v with a list in hand: they waited on the backend", d)+ }+ // the ask they started reaches the backend, once+ for deadline := time.Now().Add(10 * time.Second); asks.Load() < 2 && time.Now().Before(deadline); {+ time.Sleep(10 * time.Millisecond)+ }+ if n := asks.Load(); n != 2 {+ t.Fatalf("the backend was asked %d times, want 2: once, then once again for the stale list", n)+ }+}++// A sign-in the backend refuses is Codex's to see, and no list from+// before it is handed out in its place.+func TestCodexModelListRefusedSignInPassed(t *testing.T) {+ setup(t, provider.Chat, &fake{t: t})+ var asks atomic.Int32+ chatgpt(t, func(w http.ResponseWriter, r *http.Request) {+ if asks.Add(1) == 1 {+ w.Write([]byte(`{"models":[{"slug":"gpt-old","priority":1}]}`))+ return+ }+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(401)+ w.Write([]byte(`{"detail":"token expired"}`))+ })+ was := codexListFor+ codexListFor = time.Nanosecond+ t.Cleanup(func() { codexListFor = was })+ h := New().Handler()+ if code, has := codexListAsk(h); code != 200 || !has["gpt-old"] {+ t.Fatalf("first list %d %v", code, has)+ }+ // stale: handed out while the backend is asked again, which refuses+ codexListAsk(h)+ codexLists.asking.Wait()+ if code, _ := codexListAsk(h); code != 401 {+ t.Fatalf("after the backend refused the sign-in: %d, want its 401", code)+ }+}++// A backend that always answers after magpie stops waiting still gets its+// list to Codex: the ask goes on, and the next caller has it. Before,+// every ask was given up at codexModelsWait, so such a backend's list,+// and the version and tools its entries say, never came through (#3734).+func TestCodexModelListBackendAlwaysSlow(t *testing.T) {+ setup(t, provider.Chat, &fake{t: t})+ chatgpt(t, func(w http.ResponseWriter, r *http.Request) {+ // three times what magpie waits, well inside codexListFetch+ select {+ case <-time.After(600 * time.Millisecond):+ case <-r.Context().Done():+ return+ }+ w.Write([]byte(`{"models":[{"slug":"gpt-live","priority":1}]}`))+ })+ was := codexModelsWait+ codexModelsWait = 200 * time.Millisecond+ t.Cleanup(func() { codexModelsWait = was })+ h := New().Handler()+ for deadline := time.Now().Add(10 * time.Second); ; {+ if _, has := codexListAsk(h); has["gpt-live"] && has["fake/m1"] {+ return+ }+ if time.Now().After(deadline) {+ t.Fatal("a backend slower than magpie waits never got its list through")+ }+ time.Sleep(50 * time.Millisecond)+ }+}diff --git a/internal/gateway/codex_native_context_test.go b/internal/gateway/codex_native_context_test.goindex 29b4c51a2..49acdd919 100644--- a/internal/gateway/codex_native_context_test.go+++ b/internal/gateway/codex_native_context_test.go@@ -22,6 +22,7 @@ import (// made from Codex's cache, which keeps what magpie handed it.func TestCodexNativeModelsTakeSetWindows(t *testing.T) {codexSignedIn(t)+ forgetCodexLists(t)if err := provider.Save(provider.Provider{ID: "codex", Contexts: map[string]int{"gpt-6.1-sol": 600000, "gpt-5.5": 400000}}); err != nil {t.Fatal(err)}@@ -108,6 +109,7 @@ func TestCodexNativeModelsTakeSetWindows(t *testing.T) {{ID: "gpt-5.5", Context: 272000},})down = true+ forgetCodexLists(t) // its list, kept codexListFor, gone toogot, _ = list()if c, x := window(got["gpt-6.1-sol"]); c != 272000 || x != 872000 {t.Errorf("cached gpt-6.1-sol %d/%d, want 272000/872000", c, x)diff --git a/internal/gateway/only_picked_test.go b/internal/gateway/only_picked_test.goindex 4dbc27933..c07b7e930 100644--- a/internal/gateway/only_picked_test.go+++ b/internal/gateway/only_picked_test.go@@ -78,6 +78,7 @@ func TestOnlyPickedModelsList(t *testing.T) {// gains after it is off Codex's list until ticked.func TestCodexModelListOnlyPicked(t *testing.T) {codexSignedIn(t)+ forgetCodexLists(t)if err := provider.Save(provider.Provider{ID: "codex", Models: []string{"gpt-5.5"}}); err != nil {t.Fatal(err)}@@ -113,7 +114,10 @@ func TestCodexModelListOnlyPicked(t *testing.T) {if native := list(); len(native) != 1 || native[0] != "gpt-5.5" {t.Fatalf("switched on: %v", native)}+ // the account gains a model: the backend's list, kept codexListFor,+ // is asked againslugs += `,{"slug":"gpt-6","priority":2}`+ forgetCodexLists(t)if err := provider.Save(provider.Provider{ID: "codex", Models: []string{"gpt-5.5", "gpt-6"}}); err != nil {t.Fatal(err)}@@ -128,6 +132,7 @@ func TestCodexModelListOnlyPicked(t *testing.T) {t.Errorf("switched off: %v", native)}slugs += `,{"slug":"gpt-6-mini","priority":3}`+ forgetCodexLists(t)if err := provider.Save(provider.Provider{ID: "codex", Models: []string{"gpt-5.5", "gpt-6", "gpt-6-mini"}}); err != nil {t.Fatal(err)}