Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: Codex's model list is asked of the backend once for every app-server, and kept, so it comes within Codex's 5 s (#3734)

yetone479de8900182
diff --git a/internal/gateway/codex_backend.go b/internal/gateway/codex_backend.go
index 736058a04..6bb026739 100644
--- a/internal/gateway/codex_backend.go
+++ b/internal/gateway/codex_backend.go
@@ -14,6 +14,7 @@ import (
"regexp"
"slices"
"strings"
+ "sync"
"time"
"github.com/tidwall/gjson"
@@ -1744,40 +1745,162 @@ func copyHeaders(dst, src http.Header) {
// them, held magpie's answer past that (#539). A var so tests can say.
var codexModelsWait = 3 * time.Second
+// codexListFor is how long the backend's list is handed out again without
+// asking it; past that it is still handed out at once while it is asked
+// again (#3734). A var so tests can say.
+var codexListFor = time.Minute
+
+// codexListFetch bounds one ask of the backend, which goes on past
+// codexModelsWait for the callers after.
+const codexListFetch = 30 * time.Second
+
+// codexLists is the backend's last list for each sign-in. Twenty Codex
+// app-servers asked it at once, each on its own and each given up at
+// codexModelsWait, so a backend slower than that never got a list through
+// and Codex fell back to its own metadata, without tool_search for
+// magpie's groups, and inlined every Apps tool: ~500K tokens a turn
+// (#3734, 1,906 timeouts in a day).
+var codexLists = struct {
+ sync.Mutex
+ at map[string]*codexListed
+ // asking is the asks going, for tests to wait on
+ asking sync.WaitGroup
+}{at: map[string]*codexListed{}}
+
+type codexListed struct {
+ body []byte // the list as the backend said it, nil for none yet
+ etag string
+ got time.Time
+ // deny is a 401 or 403 the last ask got, Codex's to see
+ deny *http.Response
+ denyBody []byte
+ // asked is closed when the ask going ends; nil when none is
+ asked chan struct{}
+}
+
+// codexListKey is whose list it is: the backend's, the account's (a
+// token renewed is the same account), the version Codex says.
+func codexListKey(r *http.Request) string {
+ who := r.Header.Get("chatgpt-account-id")
+ if who == "" {
+ who = r.Header.Get("Authorization")
+ }
+ return provider.CodexBase + "\x00" + who + "\x00" + r.URL.RawQuery
+}
+
+// codexBackendList is the backend's model list for r's sign-in: the last
+// one asked when it is fresh, or at once while it is asked again; else
+// what an ask brings within codexModelsWait. One ask at a time for each
+// sign-in, whoever is waiting on it. body is nil when there is none;
+// deny, when the backend refused the sign-in.
+func (s *Server) codexBackendList(r *http.Request) (body []byte, etag string, deny *http.Response, denyBody []byte) {
+ key := codexListKey(r)
+ codexLists.Lock()
+ e := codexLists.at[key]
+ if e == nil {
+ e = &codexListed{}
+ codexLists.at[key] = e
+ }
+ if e.body != nil && time.Since(e.got) < codexListFor {
+ defer codexLists.Unlock()
+ return e.body, e.etag, nil, nil
+ }
+ if e.asked == nil {
+ // a refusal an ask before had is no answer for this one
+ e.asked, e.deny, e.denyBody = make(chan struct{}), nil, nil
+ codexLists.asking.Add(1)
+ go s.askCodexList(r.Context(), r.Header.Clone(), r.URL.RawQuery, e, e.asked)
+ }
+ asked, stale := e.asked, e.body != nil
+ codexLists.Unlock()
+ if !stale {
+ t := time.NewTimer(codexModelsWait)
+ select {
+ case <-asked:
+ case <-t.C:
+ case <-r.Context().Done():
+ }
+ t.Stop()
+ }
+ codexLists.Lock()
+ defer codexLists.Unlock()
+ return e.body, e.etag, e.deny, e.denyBody
+}
+
+// askCodexList asks the backend for its list into e, past the request
+// that started it, and closes done.
+func (s *Server) askCodexList(rctx context.Context, h http.Header, query string, e *codexListed, done chan struct{}) {
+ defer codexLists.asking.Done()
+ var body, denyBody []byte
+ var etag string
+ var deny *http.Response
+ defer func() {
+ codexLists.Lock()
+ if body != nil {
+ e.body, e.etag, e.got, e.deny, e.denyBody = body, etag, time.Now(), nil, nil
+ } else if deny != nil {
+ // a sign-in to renew: the next caller waits on the backend
+ // again and sees it, nothing stale handed out meanwhile
+ e.body, e.deny, e.denyBody = nil, deny, denyBody
+ }
+ e.asked = nil
+ codexLists.Unlock()
+ close(done)
+ }()
+ u := provider.CodexBase + "/models"
+ if query != "" {
+ u += "?" + query
+ }
+ ctx, cancel := context.WithTimeout(provider.ViaSignedIn(context.WithoutCancel(rctx), "codex"), codexListFetch)
+ defer cancel()
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+ if err != nil {
+ return
+ }
+ copyHeaders(req.Header, h)
+ // the whole list, shared by every caller, whatever one of them had
+ for _, k := range []string{"Accept-Encoding", "If-None-Match", "If-Modified-Since"} {
+ req.Header.Del(k)
+ }
+ res, err := s.client.Do(req)
+ if err != nil {
+ return
+ }
+ b, _ := io.ReadAll(io.LimitReader(res.Body, 16<<20))
+ res.Body.Close()
+ var list struct {
+ Models []any `json:"models"`
+ }
+ if res.StatusCode < 300 && json.Unmarshal(b, &list) == nil && list.Models != nil {
+ body, etag = b, res.Header.Get("ETag")
+ // the versions as the backend says them, before any is stamped
+ // (see codexcat.V1)
+ codexcat.Remember(list.Models)
+ } else if res.StatusCode == 401 || res.StatusCode == 403 {
+ deny, denyBody = &http.Response{StatusCode: res.StatusCode, Header: res.Header}, b
+ }
+}
+
// codexModels is the ChatGPT backend's model list for this sign-in, with
// magpie's models after it. Should the backend not answer, or not in
// time, Codex's last list of its own stands in.
func (s *Server) codexModels(w http.ResponseWriter, r *http.Request) {
var own []any
- etag := ""
- u := provider.CodexBase + "/models"
- if r.URL.RawQuery != "" {
- u += "?" + r.URL.RawQuery
+ body, etag, deny, denyBody := s.codexBackendList(r)
+ if deny != nil {
+ // a sign-in to renew is Codex's to see
+ w.Header().Set("Content-Type", deny.Header.Get("Content-Type"))
+ w.WriteHeader(deny.StatusCode)
+ w.Write(denyBody)
+ return
}
- ctx, cancel := context.WithTimeout(provider.ViaSignedIn(r.Context(), "codex"), codexModelsWait)
- defer cancel()
- if req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil); err == nil {
- copyHeaders(req.Header, r.Header)
- req.Header.Del("Accept-Encoding")
- if res, err := s.client.Do(req); err == nil {
- var list struct {
- Models []any `json:"models"`
- }
- b, _ := io.ReadAll(io.LimitReader(res.Body, 16<<20))
- res.Body.Close()
- if res.StatusCode < 300 && json.Unmarshal(b, &list) == nil {
- own, etag = list.Models, res.Header.Get("ETag")
- // the versions as the backend says them, before any is
- // stamped below (see codexcat.V1)
- codexcat.Remember(own)
- } else if res.StatusCode == 401 || res.StatusCode == 403 {
- // a sign-in to renew is Codex's to see
- w.Header().Set("Content-Type", res.Header.Get("Content-Type"))
- w.WriteHeader(res.StatusCode)
- w.Write(b)
- return
- }
+ if body != nil {
+ // each caller's own copy: what follows writes into the entries
+ var list struct {
+ Models []any `json:"models"`
}
+ json.Unmarshal(body, &list)
+ own = list.Models
}
// what follows reads the providers seven times over (the catalog, the
// codex provider's picks and windows, the list's tag): held, they are
diff --git a/internal/gateway/codex_backend_test.go b/internal/gateway/codex_backend_test.go
index f7b246043..7415d04d3 100644
--- a/internal/gateway/codex_backend_test.go
+++ b/internal/gateway/codex_backend_test.go
@@ -24,6 +24,7 @@ import (
func chatgpt(t *testing.T, h http.HandlerFunc) *httptest.Server {
t.Helper()
setHome(t, t.TempDir()) // no sign-ins but the test's
+ forgetCodexLists(t)
up := httptest.NewServer(h)
t.Cleanup(up.Close)
was := provider.CodexBase
diff --git a/internal/gateway/codex_models_shared_test.go b/internal/gateway/codex_models_shared_test.go
new file mode 100644
index 000000000..0dfe0e118
--- /dev/null
+++ b/internal/gateway/codex_models_shared_test.go
@@ -0,0 +1,235 @@
+package gateway
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/yetone/magpie/internal/provider"
+)
+
+// forgetCodexLists starts the test with no backend list kept, and ends it
+// with every ask of the backend it started done.
+func forgetCodexLists(t *testing.T) {
+ t.Helper()
+ reset := func() {
+ codexLists.asking.Wait()
+ codexLists.Lock()
+ codexLists.at = map[string]*codexListed{}
+ codexLists.Unlock()
+ }
+ reset()
+ t.Cleanup(reset)
+}
+
+// codexListAsk is one /models request as a Codex app-server sends it.
+func codexListAsk(h http.Handler) (int, map[string]bool) {
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest("GET", CodexPath+"/models?client_version=0.164.0", nil)
+ req.Header.Set("Authorization", "Bearer chatgpt-token")
+ req.Header.Set("chatgpt-account-id", "acct-1")
+ h.ServeHTTP(rec, req)
+ var list struct {
+ Models []map[string]any `json:"models"`
+ }
+ json.Unmarshal(rec.Body.Bytes(), &list)
+ has := map[string]bool{}
+ for _, m := range list.Models {
+ s, _ := m["slug"].(string)
+ has[s] = true
+ }
+ return rec.Code, has
+}
+
+// Twenty Codex app-servers asking for the model list at once, with a
+// ChatGPT backend slower than magpie waits for it (#3734): the backend is
+// asked once for all of them, and its list, though it came after they
+// were answered, is what the next ones get, at once, with magpie's
+// models. Before, each asked on its own and gave up at codexModelsWait,
+// so a slow backend's list never came through.
+func TestCodexModelListSharedWhenSlow(t *testing.T) {
+ setup(t, provider.Chat, &fake{t: t})
+ var asks atomic.Int32
+ hold := make(chan struct{})
+ var holdOnce sync.Once
+ letGo := func() { holdOnce.Do(func() { close(hold) }) }
+ chatgpt(t, func(w http.ResponseWriter, r *http.Request) {
+ asks.Add(1)
+ select {
+ case <-hold:
+ case <-r.Context().Done():
+ return
+ }
+ w.Write([]byte(`{"models":[{"slug":"gpt-live","priority":1}]}`))
+ })
+ t.Cleanup(letGo)
+ home, _ := os.UserHomeDir()
+ os.MkdirAll(filepath.Join(home, ".codex"), 0o755)
+ os.WriteFile(filepath.Join(home, ".codex", "models_cache.json"),
+ []byte(`{"etag":"\"v1\"","models":[{"slug":"gpt-cached","priority":1,"base_instructions":"x"}]}`), 0o644)
+ was := codexModelsWait
+ codexModelsWait = 200 * time.Millisecond
+ t.Cleanup(func() { codexModelsWait = was })
+ h := New().Handler()
+
+ // the backend held: every one answered from Codex's last list, the
+ // backend asked once
+ var wg sync.WaitGroup
+ var bad atomic.Int32
+ for range 20 {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ if code, has := codexListAsk(h); code != 200 || !has["fake/m1"] || !has["gpt-cached"] {
+ bad.Add(1)
+ }
+ }()
+ }
+ wg.Wait()
+ if bad.Load() != 0 {
+ t.Fatalf("%d of 20 answers without magpie's models or Codex's last list", bad.Load())
+ }
+ if n := asks.Load(); n != 1 {
+ t.Fatalf("the backend was asked %d times for 20 lists at once, want 1", n)
+ }
+
+ // it answers late; the ask was not given up with the first answers
+ letGo()
+ deadline := time.Now().Add(10 * time.Second)
+ for {
+ _, has := codexListAsk(h)
+ if has["gpt-live"] {
+ if !has["fake/m1"] || has["gpt-cached"] {
+ t.Fatalf("the backend's list without magpie's models, or beside the cached one: %v", has)
+ }
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("the backend's list, answered after magpie stopped waiting, never reached a caller")
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+
+ // fresh, it is handed out again without asking
+ n := asks.Load()
+ for range 5 {
+ if _, has := codexListAsk(h); !has["gpt-live"] || !has["fake/m1"] {
+ t.Fatalf("a fresh list not handed out again: %v", has)
+ }
+ }
+ if got := asks.Load() - n; got != 0 {
+ t.Fatalf("the backend was asked %d times more within codexListFor", got)
+ }
+}
+
+// Past codexListFor the last list is handed out at once while the backend
+// is asked again, so a slow backend holds no answer once it has answered
+// one; what it says then is what the callers after get.
+func TestCodexModelListStaleAnsweredAtOnce(t *testing.T) {
+ setup(t, provider.Chat, &fake{t: t})
+ var asks atomic.Int32
+ slow := make(chan struct{})
+ chatgpt(t, func(w http.ResponseWriter, r *http.Request) {
+ if asks.Add(1) == 1 {
+ w.Write([]byte(`{"models":[{"slug":"gpt-old","priority":1}]}`))
+ return
+ }
+ select {
+ case <-slow:
+ case <-r.Context().Done():
+ return
+ }
+ w.Write([]byte(`{"models":[{"slug":"gpt-new","priority":1}]}`))
+ })
+ t.Cleanup(func() { close(slow) })
+ was, wasFor := codexModelsWait, codexListFor
+ codexModelsWait, codexListFor = 3*time.Second, time.Nanosecond
+ t.Cleanup(func() { codexModelsWait, codexListFor = was, wasFor })
+ h := New().Handler()
+ if _, has := codexListAsk(h); !has["gpt-old"] {
+ t.Fatalf("first list %v", has)
+ }
+ // stale at once; the backend now holds its answer. Each answer comes
+ // from the list it has, well inside the 3 s it would have waited
+ start := time.Now()
+ for range 3 {
+ if _, has := codexListAsk(h); !has["gpt-old"] || !has["fake/m1"] {
+ t.Fatalf("a stale list not handed out: %v", has)
+ }
+ }
+ if d := time.Since(start); d >= codexModelsWait {
+ t.Fatalf("three answers took %v with a list in hand: they waited on the backend", d)
+ }
+ // the ask they started reaches the backend, once
+ for deadline := time.Now().Add(10 * time.Second); asks.Load() < 2 && time.Now().Before(deadline); {
+ time.Sleep(10 * time.Millisecond)
+ }
+ if n := asks.Load(); n != 2 {
+ t.Fatalf("the backend was asked %d times, want 2: once, then once again for the stale list", n)
+ }
+}
+
+// A sign-in the backend refuses is Codex's to see, and no list from
+// before it is handed out in its place.
+func TestCodexModelListRefusedSignInPassed(t *testing.T) {
+ setup(t, provider.Chat, &fake{t: t})
+ var asks atomic.Int32
+ chatgpt(t, func(w http.ResponseWriter, r *http.Request) {
+ if asks.Add(1) == 1 {
+ w.Write([]byte(`{"models":[{"slug":"gpt-old","priority":1}]}`))
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(401)
+ w.Write([]byte(`{"detail":"token expired"}`))
+ })
+ was := codexListFor
+ codexListFor = time.Nanosecond
+ t.Cleanup(func() { codexListFor = was })
+ h := New().Handler()
+ if code, has := codexListAsk(h); code != 200 || !has["gpt-old"] {
+ t.Fatalf("first list %d %v", code, has)
+ }
+ // stale: handed out while the backend is asked again, which refuses
+ codexListAsk(h)
+ codexLists.asking.Wait()
+ if code, _ := codexListAsk(h); code != 401 {
+ t.Fatalf("after the backend refused the sign-in: %d, want its 401", code)
+ }
+}
+
+// A backend that always answers after magpie stops waiting still gets its
+// list to Codex: the ask goes on, and the next caller has it. Before,
+// every ask was given up at codexModelsWait, so such a backend's list,
+// and the version and tools its entries say, never came through (#3734).
+func TestCodexModelListBackendAlwaysSlow(t *testing.T) {
+ setup(t, provider.Chat, &fake{t: t})
+ chatgpt(t, func(w http.ResponseWriter, r *http.Request) {
+ // three times what magpie waits, well inside codexListFetch
+ select {
+ case <-time.After(600 * time.Millisecond):
+ case <-r.Context().Done():
+ return
+ }
+ w.Write([]byte(`{"models":[{"slug":"gpt-live","priority":1}]}`))
+ })
+ was := codexModelsWait
+ codexModelsWait = 200 * time.Millisecond
+ t.Cleanup(func() { codexModelsWait = was })
+ h := New().Handler()
+ for deadline := time.Now().Add(10 * time.Second); ; {
+ if _, has := codexListAsk(h); has["gpt-live"] && has["fake/m1"] {
+ return
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("a backend slower than magpie waits never got its list through")
+ }
+ time.Sleep(50 * time.Millisecond)
+ }
+}
diff --git a/internal/gateway/codex_native_context_test.go b/internal/gateway/codex_native_context_test.go
index 29b4c51a2..49acdd919 100644
--- a/internal/gateway/codex_native_context_test.go
+++ b/internal/gateway/codex_native_context_test.go
@@ -22,6 +22,7 @@ import (
// made from Codex's cache, which keeps what magpie handed it.
func TestCodexNativeModelsTakeSetWindows(t *testing.T) {
codexSignedIn(t)
+ forgetCodexLists(t)
if err := provider.Save(provider.Provider{ID: "codex", Contexts: map[string]int{"gpt-6.1-sol": 600000, "gpt-5.5": 400000}}); err != nil {
t.Fatal(err)
}
@@ -108,6 +109,7 @@ func TestCodexNativeModelsTakeSetWindows(t *testing.T) {
{ID: "gpt-5.5", Context: 272000},
})
down = true
+ forgetCodexLists(t) // its list, kept codexListFor, gone too
got, _ = list()
if c, x := window(got["gpt-6.1-sol"]); c != 272000 || x != 872000 {
t.Errorf("cached gpt-6.1-sol %d/%d, want 272000/872000", c, x)
diff --git a/internal/gateway/only_picked_test.go b/internal/gateway/only_picked_test.go
index 4dbc27933..c07b7e930 100644
--- a/internal/gateway/only_picked_test.go
+++ b/internal/gateway/only_picked_test.go
@@ -78,6 +78,7 @@ func TestOnlyPickedModelsList(t *testing.T) {
// gains after it is off Codex's list until ticked.
func TestCodexModelListOnlyPicked(t *testing.T) {
codexSignedIn(t)
+ forgetCodexLists(t)
if err := provider.Save(provider.Provider{ID: "codex", Models: []string{"gpt-5.5"}}); err != nil {
t.Fatal(err)
}
@@ -113,7 +114,10 @@ func TestCodexModelListOnlyPicked(t *testing.T) {
if native := list(); len(native) != 1 || native[0] != "gpt-5.5" {
t.Fatalf("switched on: %v", native)
}
+ // the account gains a model: the backend's list, kept codexListFor,
+ // is asked again
slugs += `,{"slug":"gpt-6","priority":2}`
+ forgetCodexLists(t)
if err := provider.Save(provider.Provider{ID: "codex", Models: []string{"gpt-5.5", "gpt-6"}}); err != nil {
t.Fatal(err)
}
@@ -128,6 +132,7 @@ func TestCodexModelListOnlyPicked(t *testing.T) {
t.Errorf("switched off: %v", native)
}
slugs += `,{"slug":"gpt-6-mini","priority":3}`
+ forgetCodexLists(t)
if err := provider.Save(provider.Provider{ID: "codex", Models: []string{"gpt-5.5", "gpt-6", "gpt-6-mini"}}); err != nil {
t.Fatal(err)
}