Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: AutoVision and the searcher's pick keep each provider's ranked models until a list, a provider or a setting changes

yetone243aa709b290
diff --git a/internal/gateway/search.go b/internal/gateway/search.go
index 80b3bc440..d065b88df 100644
--- a/internal/gateway/search.go
+++ b/internal/gateway/search.go
@@ -451,6 +451,38 @@ func smallModel(p provider.Provider, keep func(catalog.Model) bool) string {
// recognition kept describing with an account out of its allowance, each
// image refused 429).
func smallReady(p provider.Provider, keep func(catalog.Model) bool, from provider.Protocol) (string, bool) {
+ c, ready := smallReadyOf(p, smallCands(p, keep), from)
+ return c.id, ready
+}
+
+// smallReadyOf is smallReady of cs, ranked as rankSmall ranks them: the
+// first with allowance left, each asked in turn, else the first.
+// spentModel weighs every account and key of the provider, and was asked
+// of each of its models before one was picked: a quarter of
+// AutoVision's allocations.
+func smallReadyOf(p provider.Provider, cs []smallCand, from provider.Protocol) (smallCand, bool) {
+ if len(cs) == 0 {
+ return smallCand{}, false
+ }
+ now := time.Now()
+ for _, c := range cs {
+ if _, spent := spentModel(p, c.id, from, now); !spent {
+ return c, true
+ }
+ }
+ return cs[0], false
+}
+
+// smallCand is a model smallModel may pick, and what it costs: input and
+// output a million tokens, 1e9 when it has no price.
+type smallCand struct {
+ id string
+ cost float64
+}
+
+// smallCands is the provider's models smallModel picks among, of those
+// keep holds, in the order it picks them (rankSmall).
+func smallCands(p provider.Provider, keep func(catalog.Model) bool) []smallCand {
var ms []catalog.Model
for _, m := range p.Exposed() {
if keep != nil && !keep(m) {
@@ -464,48 +496,77 @@ func smallReady(p provider.Provider, keep func(catalog.Model) bool, from provide
}
ms = append(ms, m)
}
- now := time.Now()
- ready := slices.DeleteFunc(slices.Clone(ms), func(m catalog.Model) bool {
- _, spent := spentModel(p, m.ID, from, now)
- return spent
- })
- if id := pickSmall(ready); id != "" {
- return id, true
- }
- return pickSmall(ms), false
+ return rankSmall(ms)
+}
+
+// smallKept is the providers' smallCands, kept by provider.StampNow:
+// AutoVision and the searcher's pick built every provider's list again
+// each time they were asked, for every agent's list of models.
+var smallKept stamped[smallKey, []smallCand]
+
+// smallKey is what smallCands are of: a provider (an account's, by who is
+// signed in) and which keep.
+type smallKey struct {
+ kind, id, agent, user, plan, home string
+}
+
+// keptSmallCands is smallCands, kept (smallKept); kind names keep, which
+// must hold of a model by the model and the provider alone.
+func keptSmallCands(p provider.Provider, kind string, keep func(catalog.Model) bool) []smallCand {
+ k := smallKey{kind: kind, id: p.ID}
+ if a := p.Account; a != nil {
+ k.agent, k.user, k.plan, k.home = a.Agent, a.User, a.Plan, a.Home
+ }
+ return smallKept.get(k, func() []smallCand { return smallCands(p, keep) })
}
-// pickSmall is smallModel's pick of ms.
-func pickSmall(ms []catalog.Model) string {
- // on a router, a model of the big vendors
- if big := slices.DeleteFunc(slices.Clone(ms), func(m catalog.Model) bool {
+// rankSmall is ms in the order smallModel picks them, each one's pick of
+// those after it: on a router, a model of the big vendors first; then a
+// small one; then the cheaper; then the one listed first.
+func rankSmall(ms []catalog.Model) []smallCand {
+ type ranked struct {
+ smallCand
+ router, large bool // of a vendor not big; not small
+ }
+ rs := make([]ranked, len(ms))
+ for i, m := range ms {
v, _, ok := strings.Cut(m.ID, "/")
- return ok && !slices.Contains([]string{"anthropic", "openai", "google", "x-ai"}, v)
- }); len(big) > 0 {
- ms = big
- }
- cost := func(m catalog.Model) float64 {
- if m.Price == nil {
- return 1e9
- }
- return m.Price.Input + m.Price.Output
- }
- cheapest := func(ms []catalog.Model) string {
- if len(ms) == 0 {
- return ""
- }
- return slices.MinFunc(ms, func(a, b catalog.Model) int { return cmp.Compare(cost(a), cost(b)) }).ID
- }
- small := slices.DeleteFunc(slices.Clone(ms), func(m catalog.Model) bool {
// "gemini" has a "mini" in it, which made every Gemini small and
// the first of them, a Pro, the one picked (#757)
l := strings.ReplaceAll(strings.ToLower(m.ID), "gemini", "")
- return !strings.Contains(l, "haiku") && !strings.Contains(l, "mini") && !strings.Contains(l, "flash")
+ cost := 1e9
+ if m.Price != nil {
+ cost = m.Price.Input + m.Price.Output
+ }
+ rs[i] = ranked{smallCand{m.ID, cost},
+ ok && !slices.Contains([]string{"anthropic", "openai", "google", "x-ai"}, v),
+ !strings.Contains(l, "haiku") && !strings.Contains(l, "mini") && !strings.Contains(l, "flash")}
+ }
+ slices.SortStableFunc(rs, func(a, b ranked) int {
+ if c := falseFirst(a.router, b.router); c != 0 {
+ return c
+ }
+ if c := falseFirst(a.large, b.large); c != 0 {
+ return c
+ }
+ return cmp.Compare(a.cost, b.cost)
})
- if id := cheapest(small); id != "" {
- return id
+ out := make([]smallCand, len(rs))
+ for i, r := range rs {
+ out[i] = r.smallCand
}
- return cheapest(ms)
+ return out
+}
+
+// falseFirst orders false before true.
+func falseFirst(a, b bool) int {
+ switch {
+ case a == b:
+ return 0
+ case a:
+ return 1
+ }
+ return -1
}
// searcherModel is the model p searches with when none is named: its
@@ -518,10 +579,14 @@ func searcherModel(p provider.Provider) string {
// searcherReady is searcherModel, and whether it has allowance left
// (smallReady).
func searcherReady(p provider.Provider) (string, bool) {
+ // what searchableModel says of a model is the model's and the
+ // provider's list's, which keptSmallCands is kept by
+ var keep func(catalog.Model) bool
if googleAccount(p) || p.IsRemoteMagpie() {
- return smallReady(p, func(m catalog.Model) bool { return searchableModel(p, m.ID) }, provider.Anthropic)
+ keep = func(m catalog.Model) bool { return searchableModel(p, m.ID) }
}
- return smallReady(p, nil, provider.Anthropic)
+ c, ready := smallReadyOf(p, keptSmallCands(p, "search", keep), provider.Anthropic)
+ return c.id, ready
}
// searchTool is the tool a model that can't search is given, under a name
diff --git a/internal/gateway/small_kept_test.go b/internal/gateway/small_kept_test.go
new file mode 100644
index 000000000..1f276fb1c
--- /dev/null
+++ b/internal/gateway/small_kept_test.go
@@ -0,0 +1,247 @@
+package gateway
+
+import (
+ "cmp"
+ "encoding/base64"
+ "fmt"
+ "math/rand/v2"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/yetone/magpie/internal/catalog"
+ "github.com/yetone/magpie/internal/provider"
+ "github.com/yetone/magpie/internal/settings"
+)
+
+// manyEyes is a key of a router listing n models that see, the cheapest a
+// small model of a big vendor; all of them enabled, or three. (All reads
+// the providers' file each time, which is as long as what is enabled.)
+func manyEyes(t *testing.T, n int, all bool) {
+ t.Helper()
+ fresh(t)
+ var ms []catalog.Model
+ var every []string
+ for i := range n {
+ id := fmt.Sprintf("vendor%d/model-%d", i, i)
+ if i == n-1 {
+ id = "google/gemini-3-flash"
+ }
+ ms = append(ms, catalog.Model{ID: id, Images: true, ImageInput: imageInputBool(true), Price: &catalog.Price{Input: float64(n - i), Output: 1}})
+ every = append(every, id)
+ }
+ ids := []string{ms[0].ID, ms[1].ID, "google/gemini-3-flash"}
+ if all {
+ ids = every
+ }
+ if err := provider.Save(provider.Provider{ID: "eyes", Name: "Eyes", Chat: "http://eyes.invalid/v1", Key: "k", Models: ids}); err != nil {
+ t.Fatal(err)
+ }
+ if err := catalog.SaveLive("eyes", "http://eyes.invalid/v1", ms); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// AutoVision is asked for every agent's list of models (provider.Described,
+// dsh's every 30s, the window's state), and built every provider's list
+// again each time, twice, then asked whether each of its models was out of
+// its allowance: 5.3 GB of a day's 36 GB of allocations. Asked again with
+// nothing changed, it costs the same whatever the lists' length.
+func TestAutoVisionDoesNotBuildListsAgain(t *testing.T) {
+ allocs := func(n int) float64 {
+ manyEyes(t, n, false)
+ if got := AutoVision(); got != "eyes/google/gemini-3-flash" {
+ t.Fatalf("AutoVision = %q of %d", got, n)
+ }
+ return testing.AllocsPerRun(20, func() { AutoVision() })
+ }
+ few, many := allocs(4), allocs(400)
+ t.Logf("allocations an AutoVision: %.0f with 4 models, %.0f with 400", few, many)
+ if many > few+50 {
+ t.Fatalf("AutoVision made %.0f allocations with 400 models, %.0f with 4: the lists are built again", many, few)
+ }
+ // and the searcher's pick, built the same way
+ p, err := provider.Find("eyes")
+ if err != nil {
+ t.Fatal(err)
+ }
+ searcherModel(*p)
+ if a := testing.AllocsPerRun(20, func() { searcherModel(*p) }); a > 50 {
+ t.Fatalf("searcherModel made %.0f allocations with 400 models", a)
+ }
+}
+
+// What AutoVision keeps is built again when what it was built of changes:
+// a provider saved (its picks), a vendor's list fetched, an account signed
+// in, the settings and the providers' file written by another magpie.
+func TestAutoVisionSeesEachChange(t *testing.T) {
+ routerUp(t, "text")
+ if m := AutoVision(); m != "" {
+ t.Fatalf("AutoVision = %q with no model that sees enabled", m)
+ }
+
+ // a provider saved: Haiku enabled
+ p, err := provider.Find("router")
+ if err != nil {
+ t.Fatal(err)
+ }
+ p.Models = []string{"text", "anthropic/claude-haiku-5.5", "google/gemini-3-flash"}
+ if err := provider.Save(*p); err != nil {
+ t.Fatal(err)
+ }
+ if m := AutoVision(); m != "router/google/gemini-3-flash" {
+ t.Fatalf("AutoVision = %q after Haiku and Gemini were enabled", m)
+ }
+
+ // the vendor's list fetched anew: Haiku the cheaper now
+ if err := catalog.SaveLive("router", p.Chat, []catalog.Model{
+ {ID: "text", ImageInput: imageInputBool(false), Price: &catalog.Price{Input: 0.1, Output: 0.1}},
+ {ID: "anthropic/claude-haiku-5.5", Images: true, ImageInput: imageInputBool(true), Price: &catalog.Price{Input: 0.1, Output: 0.2}},
+ {ID: "google/gemini-3-flash", Images: true, ImageInput: imageInputBool(true), Price: &catalog.Price{Input: 0.3, Output: 2.5}},
+ }); err != nil {
+ t.Fatal(err)
+ }
+ if m := AutoVision(); m != "router/anthropic/claude-haiku-5.5" {
+ t.Fatalf("AutoVision = %q after the list priced Haiku the cheaper", m)
+ }
+
+ // the providers' file written by another magpie (magpie provider
+ // edit in a terminal): Haiku taken out
+ b, err := os.ReadFile(provider.Path())
+ if err != nil {
+ t.Fatal(err)
+ }
+ other := strings.Replace(string(b), `"anthropic/claude-haiku-5.5",`, "", 1)
+ if other == string(b) {
+ t.Fatalf("no Haiku pick in %s", b)
+ }
+ if err := os.WriteFile(provider.Path(), []byte(other), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if m := AutoVision(); m != "router/google/gemini-3-flash" {
+ t.Fatalf("AutoVision = %q after another magpie took Haiku out", m)
+ }
+
+ // a setting saved moves what it is kept by
+ before := provider.StampNow()
+ st := settings.Load()
+ st.Searcher = "off"
+ if err := settings.Save(st); err != nil {
+ t.Fatal(err)
+ }
+ if provider.StampNow() == before {
+ t.Fatal("a setting saved left the stamp where it was")
+ }
+
+ // an account signed in, in this same home: Codex's tier goes first
+ if err := catalog.SaveLive("codex", provider.CodexBase, []catalog.Model{{ID: "gpt-6.1-sol", Images: true, ImageInput: imageInputBool(true)}}); err != nil {
+ t.Fatal(err)
+ }
+ home, _ := os.UserHomeDir()
+ os.MkdirAll(filepath.Join(home, ".codex"), 0o755)
+ claims := func(m map[string]any) string {
+ return "h." + base64.RawURLEncoding.EncodeToString(mustJSON(m)) + ".s"
+ }
+ auth := map[string]any{"auth_mode": "chatgpt", "tokens": map[string]any{
+ "id_token": claims(map[string]any{"email": "[email protected]"}),
+ "access_token": claims(map[string]any{"exp": time.Now().Add(time.Hour).Unix(), "who": "acct-1"}),
+ "refresh_token": "r-acct-1", "account_id": "acct-1"}}
+ if err := os.WriteFile(filepath.Join(home, ".codex", "auth.json"), mustJSON(auth), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ provider.ForgetAccounts() // the 30s accounts are looked for again in
+ t.Cleanup(provider.ForgetAccounts)
+ if m := AutoVision(); m != "codex/gpt-6.1-sol" {
+ t.Fatalf("AutoVision = %q after Codex was signed in", m)
+ }
+}
+
+// A model out of its allowance is passed over as soon as it is, though
+// the lists are kept: what is out is asked each time.
+func TestAutoVisionKeptListsAskAllowanceEachTime(t *testing.T) {
+ codexSignedIn(t)
+ used := map[string]float64{"[email protected]": 20}
+ capUsage(t, used)
+ if err := catalog.SaveLive("codex", provider.CodexBase, []catalog.Model{{ID: "gpt-6.1-sol", Images: true, ImageInput: imageInputBool(true)}}); err != nil {
+ t.Fatal(err)
+ }
+ if got, ready := autoVision(); got != "codex/gpt-6.1-sol" || !ready {
+ t.Fatalf("autoVision = %q %v", got, ready)
+ }
+ used["[email protected]"] = 100
+ if got, ready := autoVision(); got != "codex/gpt-6.1-sol" || ready {
+ t.Fatalf("autoVision = %q %v with the only account used up", got, ready)
+ }
+}
+
+// rankSmall's order is pickSmall's pick of every subset, as pickSmall
+// was before the ranking: the first of a subset is its pick.
+func TestRankSmallIsPickSmall(t *testing.T) {
+ pickSmall := func(ms []catalog.Model) string {
+ if big := slices.DeleteFunc(slices.Clone(ms), func(m catalog.Model) bool {
+ v, _, ok := strings.Cut(m.ID, "/")
+ return ok && !slices.Contains([]string{"anthropic", "openai", "google", "x-ai"}, v)
+ }); len(big) > 0 {
+ ms = big
+ }
+ cost := func(m catalog.Model) float64 {
+ if m.Price == nil {
+ return 1e9
+ }
+ return m.Price.Input + m.Price.Output
+ }
+ cheapest := func(ms []catalog.Model) string {
+ if len(ms) == 0 {
+ return ""
+ }
+ return slices.MinFunc(ms, func(a, b catalog.Model) int { return cmp.Compare(cost(a), cost(b)) }).ID
+ }
+ small := slices.DeleteFunc(slices.Clone(ms), func(m catalog.Model) bool {
+ l := strings.ReplaceAll(strings.ToLower(m.ID), "gemini", "")
+ return !strings.Contains(l, "haiku") && !strings.Contains(l, "mini") && !strings.Contains(l, "flash")
+ })
+ if id := cheapest(small); id != "" {
+ return id
+ }
+ return cheapest(ms)
+ }
+ r := rand.New(rand.NewPCG(1, 2))
+ vendors := []string{"", "anthropic/", "openai/", "google/", "x-ai/", "meta/", "qwen/"}
+ names := []string{"claude-haiku-5", "gpt-6-mini", "gemini-3-flash", "gemini-3-pro", "big", "o9", "llama"}
+ for range 2000 {
+ var ms []catalog.Model
+ for i := range r.IntN(8) + 1 {
+ m := catalog.Model{ID: fmt.Sprintf("%s%s-%d", vendors[r.IntN(len(vendors))], names[r.IntN(len(names))], i)}
+ if r.IntN(4) > 0 {
+ m.Price = &catalog.Price{Input: float64(r.IntN(3)), Output: float64(r.IntN(3))}
+ }
+ ms = append(ms, m)
+ }
+ cs := rankSmall(ms)
+ if cs[0].id != pickSmall(ms) {
+ t.Fatalf("rankSmall's first %q, pickSmall %q of %v", cs[0].id, pickSmall(ms), ms)
+ }
+ // a subset: those not spent
+ keep := map[string]bool{}
+ var sub []catalog.Model
+ for _, m := range ms {
+ if r.IntN(2) == 0 {
+ keep[m.ID] = true
+ sub = append(sub, m)
+ }
+ }
+ first := ""
+ for _, c := range cs {
+ if keep[c.id] {
+ first = c.id
+ break
+ }
+ }
+ if first != pickSmall(sub) {
+ t.Fatalf("rankSmall's first of a subset %q, pickSmall %q of %v", first, pickSmall(sub), sub)
+ }
+ }
+}
diff --git a/internal/gateway/stamped.go b/internal/gateway/stamped.go
new file mode 100644
index 000000000..0c78ba71c
--- /dev/null
+++ b/internal/gateway/stamped.go
@@ -0,0 +1,51 @@
+package gateway
+
+import (
+ "sync"
+ "time"
+
+ "github.com/yetone/magpie/internal/provider"
+)
+
+// stampedFor is how long what a stamped keeps is used while the stamp
+// stays the same: a list another magpie fetched, or an account another
+// program signed in to, moves no stamp, and is seen this long after.
+const stampedFor = time.Minute
+
+// stamped keeps what is built from the providers' model lists, by key,
+// until provider.StampNow moves or stampedFor has gone by. Each
+// AutoVision (every agent's model list asks it, provider.Described) built
+// every provider's list again, twice, and a probe of the gateway (info)
+// the whole catalog to count it: 5.3 GB and 5.4 GB of a day's 36 GB of
+// allocations (yg6888 on X, ~1 GB held). Two callers building one key at
+// once each build it; none waits on another.
+type stamped[K comparable, V any] struct {
+ mu sync.Mutex
+ stamp provider.Stamp
+ at time.Time
+ of map[K]V
+}
+
+func (s *stamped[K, V]) get(k K, build func() V) V {
+ st := provider.StampNow()
+ s.mu.Lock()
+ if s.stamp != st || time.Since(s.at) >= stampedFor {
+ s.stamp, s.at, s.of = st, time.Now(), nil
+ }
+ if v, ok := s.of[k]; ok {
+ s.mu.Unlock()
+ return v
+ }
+ s.mu.Unlock()
+ // built from what is there after st was taken: never older than st
+ v := build()
+ s.mu.Lock()
+ if s.stamp == st {
+ if s.of == nil {
+ s.of = map[K]V{}
+ }
+ s.of[k] = v
+ }
+ s.mu.Unlock()
+ return v
+}
diff --git a/internal/gateway/vision.go b/internal/gateway/vision.go
index cefc225d7..7a2859cde 100644
--- a/internal/gateway/vision.go
+++ b/internal/gateway/vision.go
@@ -228,7 +228,7 @@ func autoVision() (string, bool) {
if bestReady && t > bestTier {
continue
}
- m, cost, ready := smallModelWhere(p, sees)
+ m, cost, ready := smallModelWhere(p, "sees", sees)
if m == "" {
continue
}
@@ -468,16 +468,11 @@ func (s *Server) seenBody(ctx context.Context, proto provider.Protocol, body []b
// smallModelWhere is smallModel among the provider's models that keep
// holds, what it costs, and whether it has allowance left (smallReady);
-// "" when there is none.
-func smallModelWhere(p provider.Provider, keep func(catalog.Model) bool) (string, float64, bool) {
- id, ready := smallReady(p, keep, provider.Chat)
- if id == "" {
+// "" when there is none. kind names keep (keptSmallCands).
+func smallModelWhere(p provider.Provider, kind string, keep func(catalog.Model) bool) (string, float64, bool) {
+ c, ready := smallReadyOf(p, keptSmallCands(p, kind, keep), provider.Chat)
+ if c.id == "" {
return "", 0, false
}
- for _, m := range p.Exposed() {
- if m.ID == id && m.Price != nil {
- return id, m.Price.Input + m.Price.Output, ready
- }
- }
- return id, 1e9, ready
+ return c.id, c.cost, ready
}