Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

provider: the catalog a request built is kept for the next requests while nothing it is built from moved

yetone6a050eebe669
diff --git a/internal/provider/entries_kept_test.go b/internal/provider/entries_kept_test.go
new file mode 100644
index 000000000..b1fc349f6
--- /dev/null
+++ b/internal/provider/entries_kept_test.go
@@ -0,0 +1,98 @@
+package provider
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+// Requests further apart than heldFor resolve their models from the
+// catalog the first one built while nothing it is built from moved
+// (StampNow): each built it again from every provider's models and
+// accounts, ~725 MB of allocations in 300s of a gateway in use. A
+// provider saved, or keptFor gone by, has it built again, and the next
+// request sees what was saved. The groups built from it are kept with it.
+func TestCatalogKeptAcrossRequests(t *testing.T) {
+ isolate(t)
+ home := t.TempDir()
+ t.Setenv("HOME", home)
+ t.Setenv("USERPROFILE", home)
+ t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config"))
+ t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache"))
+ defer func(d, k time.Duration) { heldFor, keptFor = d, k }(heldFor, keptFor)
+ heldFor = 0 // each request after the last one's hold has gone
+ keptFor = time.Minute // a loaded machine's pause isn't the bound
+ if err := Save(Provider{ID: "relay", Name: "Relay", Key: "k", Chat: "http://127.0.0.1:1/v1", Models: []string{"m1", "m2"}}); err != nil {
+ t.Fatal(err)
+ }
+ // as a request to the gateway: the model resolved, and the served
+ // list (groups and all) read for its reply limit
+ request := func(id string) bool {
+ defer Hold()()
+ _, _, ok := Resolve(id)
+ Served()
+ return ok
+ }
+ if !request("relay/m1") {
+ t.Fatal("relay/m1 not resolved")
+ }
+ before, groups := entriesBuilt.Load(), groupsBuilt.Load()
+ for range 20 {
+ if !request("relay/m2") {
+ t.Fatal("relay/m2 not resolved")
+ }
+ }
+ if n := entriesBuilt.Load() - before; n != 0 {
+ t.Fatalf("20 requests with nothing changed built the catalog %d times, want 0", n)
+ }
+ if n := groupsBuilt.Load() - groups; n != 0 {
+ t.Fatalf("20 requests with nothing changed built the groups %d times, want 0", n)
+ }
+
+ // a provider saved: the next request sees it
+ if err := Save(Provider{ID: "relay2", Name: "Relay 2", Key: "k", Chat: "http://127.0.0.1:1/v1", Models: []string{"m3"}}); err != nil {
+ t.Fatal(err)
+ }
+ if !request("relay2/m3") {
+ t.Fatal("a provider saved isn't resolved by the next request")
+ }
+ if e := entriesOfID(t, "relay2/m3"); !e {
+ t.Fatal("relay2/m3 not in the served catalog after the save")
+ }
+
+ // providers.json written by another process (the CLI): seen at once
+ b, err := os.ReadFile(Path())
+ if err != nil {
+ t.Fatal(err)
+ }
+ before = entriesBuilt.Load()
+ if err := os.WriteFile(Path(), append(b, '\n'), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ request("relay/m1")
+ if n := entriesBuilt.Load() - before; n != 1 {
+ t.Fatalf("after another process wrote providers.json: built %d times, want 1", n)
+ }
+
+ // keptFor gone by: built again, for what moves no stamp
+ keptFor = 0
+ before = entriesBuilt.Load()
+ request("relay/m1")
+ request("relay/m1")
+ if n := entriesBuilt.Load() - before; n < 2 {
+ t.Fatalf("past keptFor: built %d times in 2 requests, want one each at least", n)
+ }
+
+}
+
+func entriesOfID(t *testing.T, id string) bool {
+ t.Helper()
+ defer Hold()()
+ for _, e := range Served() {
+ if e.ID == id {
+ return true
+ }
+ }
+ return false
+}
diff --git a/internal/provider/hold.go b/internal/provider/hold.go
index 1e5314543..1076d7a4b 100644
--- a/internal/provider/hold.go
+++ b/internal/provider/hold.go
@@ -5,6 +5,7 @@ import (
"os"
"slices"
"sync"
+ "sync/atomic"
"time"
"github.com/yetone/magpie/internal/catalog"
@@ -206,7 +207,79 @@ func heldOf[T any](key string, build func() T) T {
func heldEntries(build func() []Entry) []Entry {
// clipped: a caller's append copies
- return heldOf("entries", func() []Entry { return slices.Clip(build()) })
+ return heldOf("entries", func() []Entry {
+ return keptOf("entries", func() []Entry {
+ entriesBuilt.Add(1)
+ return slices.Clip(build())
+ })
+ })
+}
+
+// entriesBuilt counts the catalogs built through heldEntries, for a test
+// that counts them.
+var entriesBuilt atomic.Int64
+
+// keptFor is how long what kept has is used while the Stamp stays the
+// same: an account another program signed in to or out of, or a model
+// retired by its date, moves no stamp, and is seen this long after.
+var keptFor = time.Minute
+
+// kept is what a request built from every provider's models (the catalog,
+// the switched-off providers' models, the groups and their entries), for
+// the requests after it while StampNow stays the same and keptFor hasn't
+// gone by. Held for heldFor alone, an agent's requests a few seconds apart
+// each built them all again from every provider's models and accounts
+// (~725 MB of allocations in 300s of a gateway in use). What it gives is
+// shared, as what heldOf gives: only for reading. Two requests building
+// one at once each build it; none waits on another.
+var kept struct {
+ sync.Mutex
+ stamp Stamp
+ at time.Time
+ of map[string]any
+}
+
+// keptOf is what build gives, kept across requests (kept) while one holds
+// the catalog, and built afresh outside one.
+func keptOf[T any](key string, build func() T) T {
+ return keptFit(key, func(any) bool { return true }, build)
+}
+
+// keptFit is keptOf, taking what is kept only where fits says it is
+// still the one wanted.
+func keptFit[T any](key string, fits func(any) bool, build func() T) T {
+ if !holdingNow() {
+ return build()
+ }
+ st := StampNow()
+ kept.Lock()
+ if kept.stamp != st || time.Since(kept.at) >= keptFor {
+ kept.stamp, kept.at, kept.of = st, time.Now(), nil
+ }
+ if v, ok := kept.of[key]; ok && fits(v) {
+ kept.Unlock()
+ return v.(T)
+ }
+ kept.Unlock()
+ // built from what is there after st was taken: never older than st,
+ // and a write meanwhile moves the stamp the next look takes
+ v := build()
+ kept.Lock()
+ if kept.stamp == st {
+ if kept.of == nil {
+ kept.of = map[string]any{}
+ }
+ kept.of[key] = v
+ }
+ kept.Unlock()
+ return v
+}
+
+// holdingNow is whether a request holds the catalog now.
+func holdingNow() bool {
+ held.Lock()
+ defer held.Unlock()
+ return held.holds > 0
}
// heldPlugins is plugin.Cached, which reads the plugins' list and
@@ -248,13 +321,17 @@ func heldFrom[T any](key string, entries []Entry, build func([]Entry) T) T {
}
gen := held.gen
held.Unlock()
- v := build(entries)
+ f := keptFit(key, func(v any) bool {
+ f, ok := v.(fromEntries[T])
+ return ok && f.at == at && f.n == len(entries)
+ }, func() fromEntries[T] { return fromEntries[T]{at, len(entries), build(entries)} })
+ v := f.v
held.Lock()
if held.holds > 0 && held.gen == gen {
if held.built == nil {
held.built = map[string]heldValue{}
}
- held.built[key] = heldValue{fromEntries[T]{at, len(entries), v}, time.Now()}
+ held.built[key] = heldValue{f, time.Now()}
}
held.Unlock()
return v
diff --git a/internal/provider/models.go b/internal/provider/models.go
index 07771049e..ff394a81c 100644
--- a/internal/provider/models.go
+++ b/internal/provider/models.go
@@ -2431,7 +2431,9 @@ func entriesOff(off func(pid, model string) bool) []Entry {
// plugin not signed in, finds no group.
func heldOffEntries() []Entry {
return heldOf("off entries", func() []Entry {
- return slices.Clip(entriesOf(func(p Provider) bool { return p.Ready() && p.Off }, disabledNow()))
+ return keptOf("off entries", func() []Entry {
+ return slices.Clip(entriesOf(func(p Provider) bool { return p.Ready() && p.Off }, disabledNow()))
+ })
})
}