Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

provider: a request to a routing group builds the groups' entries once, not once per look at every member of every group (colmugx, Yayoi_no_yume on X)

yetonea595f1559c35
diff --git a/internal/gateway/group_overhead_bench_test.go b/internal/gateway/group_overhead_bench_test.go
new file mode 100644
index 000000000..9540b1127
--- /dev/null
+++ b/internal/gateway/group_overhead_bench_test.go
@@ -0,0 +1,196 @@
+package gateway
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "io/fs"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/yetone/magpie/internal/provider"
+ "github.com/yetone/magpie/internal/settings"
+)
+
+// groupBenchUpstream answers an Anthropic request as it asked: streamed
+// when its body says so. Refusing, it answers code with an overloaded
+// error.
+type groupBenchUpstream struct{ code int }
+
+const groupBenchReply = `{"id":"msg_1","type":"message","role":"assistant","model":"m1","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":7,"output_tokens":1}}`
+
+var groupBenchStream = sse(
+ `event: message_start`+"\n"+`data: {"type":"message_start","message":{"id":"msg_1","model":"m1","usage":{"input_tokens":7}}}`,
+ `event: content_block_start`+"\n"+`data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`,
+ `event: content_block_delta`+"\n"+`data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ok"}}`,
+ `event: content_block_stop`+"\n"+`data: {"type":"content_block_stop","index":0}`,
+ `event: message_delta`+"\n"+`data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1}}`,
+ `event: message_stop`+"\n"+`data: {"type":"message_stop"}`)
+
+func (u groupBenchUpstream) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ b, _ := io.ReadAll(r.Body)
+ if u.code != 0 {
+ w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("Retry-After", "3600")
+ w.WriteHeader(u.code)
+ io.WriteString(w, `{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}`)
+ return
+ }
+ if bytes.Contains(b, []byte(`"stream":true`)) {
+ w.Header().Set("Content-Type", "text/event-stream")
+ io.WriteString(w, groupBenchStream)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ io.WriteString(w, groupBenchReply)
+}
+
+// groupBenchCatalog is a catalog of providers serving models each, the
+// same model ids at each (so magpie finds a group of each model), and
+// groups of one, four and eight members, one whose first member fails
+// every time, and one whose first member is resting.
+func groupBenchCatalog(b *testing.B, providers, models int) {
+ b.Helper()
+ b.Setenv("XDG_CONFIG_HOME", b.TempDir())
+ b.Setenv("XDG_CACHE_HOME", b.TempDir())
+ good := httptest.NewServer(groupBenchUpstream{})
+ b.Cleanup(good.Close)
+ failing := httptest.NewServer(groupBenchUpstream{code: 500})
+ b.Cleanup(failing.Close)
+ resting := httptest.NewServer(groupBenchUpstream{code: 429})
+ b.Cleanup(resting.Close)
+ var ids []string
+ for i := range models {
+ ids = append(ids, fmt.Sprintf("m%d", i))
+ }
+ for i := range providers {
+ p := provider.Provider{ID: fmt.Sprintf("p%d", i), Name: fmt.Sprintf("P%d", i), Key: "k", Models: ids, Anthropic: good.URL}
+ if err := provider.Save(p); err != nil {
+ b.Fatal(err)
+ }
+ }
+ for _, p := range []provider.Provider{
+ {ID: "bad", Name: "Bad", Key: "k", Models: ids, Anthropic: failing.URL},
+ {ID: "rest", Name: "Rest", Key: "k", Models: ids, Anthropic: resting.URL},
+ } {
+ if err := provider.Save(p); err != nil {
+ b.Fatal(err)
+ }
+ }
+ if err := settings.Save(settings.Settings{}); err != nil {
+ b.Fatal(err)
+ }
+ for _, g := range []provider.Group{
+ {ID: "g1", Name: "G1", Members: []string{"p0/m1"}, Routing: provider.Ordered},
+ {ID: "g4", Name: "G4", Members: []string{"p0/m1", "p1/m1", "p2/m1", "p3/m1"}, Routing: provider.Ordered},
+ {ID: "g8u", Name: "G8U", Members: []string{"p0/m1", "p1/m1", "p2/m1", "p3/m1", "p4/m1", "p5/m1", "p6/m1", "p7/m1"}, Routing: provider.LeastUsed},
+ {ID: "gfail", Name: "GFail", Members: []string{"bad/m1", "p0/m1", "p1/m1"}, Routing: provider.Ordered},
+ {ID: "grest", Name: "GRest", Members: []string{"rest/m1", "p0/m1", "p1/m1"}, Routing: provider.Ordered},
+ } {
+ if err := provider.SaveGroup(g); err != nil {
+ b.Fatal(err)
+ }
+ }
+ // as a user's files are: written a while ago, not just now (a file
+ // written in the last seconds is read again each time)
+ old := time.Now().Add(-time.Hour)
+ filepath.WalkDir(os.Getenv("XDG_CONFIG_HOME"), func(path string, _ fs.DirEntry, _ error) error {
+ return os.Chtimes(path, old, old)
+ })
+ provider.Changed()
+}
+
+// groupBenchCase is one benchmark's request.
+type groupBenchCase struct {
+ providers, models, words int
+ stream bool
+ model string
+}
+
+func (c groupBenchCase) name() string {
+ return fmt.Sprintf("catalog=%dx%d/%dKB/stream=%v/%s", c.providers, c.models, c.words*4/1000, c.stream, c.model)
+}
+
+// groupBenchCases: a dozen providers of 20 models, with a ~200KB and a
+// ~2MB conversation, and a large catalog (30 providers of 60 models, as
+// a user of several relays has) with the ~200KB one.
+func groupBenchCases() []groupBenchCase {
+ var out []groupBenchCase
+ for _, size := range []struct{ providers, models, words int }{{12, 20, 50_000}, {12, 20, 500_000}, {30, 60, 50_000}} {
+ for _, stream := range []bool{false, true} {
+ for _, model := range []string{"p0/m1", "group/g1", "group/g4", "group/g8u", "group/gfail", "group/grest"} {
+ out = append(out, groupBenchCase{size.providers, size.models, size.words, stream, model})
+ }
+ }
+ }
+ return out
+}
+
+// groupBenchServer is the gateway with c's catalog, and a request of the
+// body c sends that fails the benchmark unless it is answered.
+func groupBenchServer(b *testing.B, c groupBenchCase) func() {
+ noRests()
+ b.Cleanup(noRests)
+ groupBenchCatalog(b, c.providers, c.models)
+ body := largeConversation(provider.Anthropic, c.model, c.words)
+ if c.stream {
+ body = bytes.Replace(body, []byte(`"stream":false`), []byte(`"stream":true`), 1)
+ }
+ b.SetBytes(int64(len(body)))
+ h := New().Handler()
+ do := func() {
+ rec := httptest.NewRecorder()
+ h.ServeHTTP(rec, httptest.NewRequest("POST", "/v1/messages", bytes.NewReader(body)))
+ if rec.Code != 200 || !bytes.Contains(rec.Body.Bytes(), []byte("ok")) {
+ b.Fatalf("%d %s", rec.Code, rec.Body.String())
+ }
+ }
+ do() // the resting member rests
+ return do
+}
+
+// BenchmarkGroupOverhead is a request to a provider's model and the same
+// request to routing groups, streamed and not, with a long conversation:
+// what magpie adds for a group is the difference. cpu-ms/op is the CPU
+// the process spent, which a busy machine doesn't change as it does the
+// wall time.
+func BenchmarkGroupOverhead(b *testing.B) {
+ for _, c := range groupBenchCases() {
+ b.Run(c.name(), func(b *testing.B) {
+ do := groupBenchServer(b, c)
+ b.ResetTimer()
+ cpu, _ := processCPU()
+ for range b.N {
+ do()
+ }
+ after, _ := processCPU()
+ b.ReportMetric(float64(after-cpu)/float64(b.N)/1e6, "cpu-ms/op")
+ })
+ }
+}
+
+// BenchmarkGroupOverheadParallel is BenchmarkGroupOverhead's requests sent
+// at once, as several agents send them.
+func BenchmarkGroupOverheadParallel(b *testing.B) {
+ for _, c := range groupBenchCases() {
+ if c.stream || c.words > 50_000 {
+ continue
+ }
+ b.Run(c.name(), func(b *testing.B) {
+ do := groupBenchServer(b, c)
+ b.ResetTimer()
+ cpu, _ := processCPU()
+ b.RunParallel(func(pb *testing.PB) {
+ for pb.Next() {
+ do()
+ }
+ })
+ after, _ := processCPU()
+ b.ReportMetric(float64(after-cpu)/float64(b.N)/1e6, "cpu-ms/op")
+ })
+ }
+}
diff --git a/internal/provider/account_test.go b/internal/provider/account_test.go
index 3230a4d2c..af677c910 100644
--- a/internal/provider/account_test.go
+++ b/internal/provider/account_test.go
@@ -68,7 +68,7 @@ func signIn(t *testing.T) string {
// isolate keeps tests off the machine's own Claude Code Keychain
// logins and forgets anything a previous test cached.
-func isolate(t *testing.T) {
+func isolate(t testing.TB) {
t.Helper()
oldKeychain, oldBase, oldExe := claudeKeychain, claudeBase, claudeExecutable
oldDevin := DevinExecutable
diff --git a/internal/provider/disabled.go b/internal/provider/disabled.go
index b35585807..724d14f98 100644
--- a/internal/provider/disabled.go
+++ b/internal/provider/disabled.go
@@ -34,6 +34,22 @@ var lastDisabled struct {
// disabledList is the patterns in force, and unknown when settings.json
// couldn't be read and none was read before.
func disabledList() (list []string, unknown bool) {
+ // read once while a request holds the catalog: each group's members
+ // (membersIn) asked for it, and parsed settings.json each time
+ r := heldOf("disabled", func() disabledRead {
+ list, unknown := readDisabled()
+ return disabledRead{slices.Clip(list), unknown}
+ })
+ return r.list, r.unknown
+}
+
+type disabledRead struct {
+ list []string
+ unknown bool
+}
+
+// readDisabled is disabledList read from settings.json now.
+func readDisabled() (list []string, unknown bool) {
s, err := settings.LoadChecked()
lastDisabled.Lock()
defer lastDisabled.Unlock()
diff --git a/internal/provider/group.go b/internal/provider/group.go
index d641b5dd0..feba98c31 100644
--- a/internal/provider/group.go
+++ b/internal/provider/group.go
@@ -868,10 +868,34 @@ func membersOff(entries []Entry, all []Group, g Group, off func(pid, model strin
// every member fixed, the group offers the levels they are fixed at, so
// that an agent still asks it to reason. A group that names its own
// levels (Group.Levels) offers those, and so does a group in it for its
-// models.
+// models. Held while a request holds the catalog (heldFrom).
func groupEntries(entries []Entry) []Entry {
+ return heldFrom("group entries", entries, buildGroupEntries)
+}
+
+// entryOf is where in entries the last entry of provider pid's model is,
+// -1 when none is.
+type entryOf func(pid, model string) int
+
+// entriesBy is entryOf for entries, looked up in a map built once rather
+// than by a pass over every entry for each member of each group.
+func entriesBy(entries []Entry) entryOf {
+ at := make(map[[2]string]int, len(entries))
+ for i := range entries {
+ at[[2]string{entries[i].Provider.ID, entries[i].Model}] = i
+ }
+ return func(pid, model string) int {
+ if i, ok := at[[2]string{pid, model}]; ok {
+ return i
+ }
+ return -1
+ }
+}
+
+func buildGroupEntries(entries []Entry) []Entry {
var out []Entry
all := groupsIn(entries)
+ find := entriesBy(entries)
for _, g := range all {
if g.Hidden || g.Disabled {
continue
@@ -899,11 +923,10 @@ func groupEntries(entries []Entry) []Entry {
images, thinks, ctx, output := false, false, 0, 0
var imageInput *bool
unknown, reasonless := false, false
- for _, x := range entries {
- if x.Provider.ID == m.Provider.ID && x.Model == m.Model {
- efforts, images, thinks, ctx, output, imageInput = x.Efforts, x.Images, x.Reasoning, x.Context, x.Output, x.ImageInput
- unknown, reasonless = levelsUnknown(x), x.Reasonless
- }
+ if j := find(m.Provider.ID, m.Model); j >= 0 {
+ x := &entries[j]
+ efforts, images, thinks, ctx, output, imageInput = x.Efforts, x.Images, x.Reasoning, x.Context, x.Output, x.ImageInput
+ unknown, reasonless = levelsUnknown(*x), x.Reasonless
}
// it reasons when a member does: the gateway sends the
// effort to the members that take it, and a member known not
@@ -1011,8 +1034,13 @@ func groupEntries(entries []Entry) []Entry {
ruledEntry(&e, g.Live(), ms, entries)
if g.Context > 0 {
e.Context, e.WindowSet = g.Context, true
- } else if smallest, typed := smallestWindow(entries, ms); g.Context == ContextSmallest && smallest > 0 {
- e.Context, e.WindowSet = smallest, typed
+ } else if g.Context == ContextSmallest {
+ // only for a group that follows its smallest member: run for
+ // every group, it was another pass over the catalog for each
+ // member of each (8f4d091c6)
+ if smallest, typed := smallestWindowBy(entries, find, ms); smallest > 0 {
+ e.Context, e.WindowSet = smallest, typed
+ }
}
e.Family = g.Family
out = append(out, e)
@@ -1029,12 +1057,14 @@ func groupEntries(entries []Entry) []Entry {
// group set to 500K itself wasn't). A tie counts as typed: the window is
// the user's either way.
func smallestWindow(entries []Entry, ms []Member) (n int, typed bool) {
+ return smallestWindowBy(entries, entriesBy(entries), ms)
+}
+
+func smallestWindowBy(entries []Entry, find entryOf, ms []Member) (n int, typed bool) {
for _, m := range ms {
ctx, set := 0, false
- for _, x := range entries {
- if x.Provider.ID == m.Provider.ID && x.Model == m.Model {
- ctx, set = x.Context, x.WindowSet
- }
+ if j := find(m.Provider.ID, m.Model); j >= 0 {
+ ctx, set = entries[j].Context, entries[j].WindowSet
}
switch {
case ctx <= 0:
diff --git a/internal/provider/group_held_test.go b/internal/provider/group_held_test.go
new file mode 100644
index 000000000..819447c43
--- /dev/null
+++ b/internal/provider/group_held_test.go
@@ -0,0 +1,136 @@
+package provider
+
+import (
+ "fmt"
+ "path/filepath"
+ "testing"
+
+ "github.com/yetone/magpie/internal/settings"
+)
+
+// groupHeldFixture is a dozen providers serving the same models, so
+// magpie finds a group of each, and a group of the user's of four.
+func groupHeldFixture(t *testing.T) {
+ t.Helper()
+ isolate(t)
+ home := t.TempDir()
+ t.Setenv("HOME", home)
+ t.Setenv("USERPROFILE", home)
+ t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config"))
+ t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache"))
+ t.Setenv("MAGPIE_NO_ACCOUNTS", "1")
+ var models []string
+ for i := range 10 {
+ models = append(models, fmt.Sprintf("m%d", i))
+ }
+ for i := range 12 {
+ if err := Save(Provider{ID: fmt.Sprintf("p%d", i), Name: fmt.Sprintf("P%d", i), Key: "k", Models: models, Anthropic: "http://127.0.0.1:9"}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := settings.Save(settings.Settings{DisabledModels: []string{"p11/m9"}}); err != nil {
+ t.Fatal(err)
+ }
+ if err := SaveGroup(Group{ID: "g4", Name: "G4", Members: []string{"p0/m1", "p1/m1", "p2/m1", "p3/m1"}, Routing: Ordered}); err != nil {
+ t.Fatal(err)
+ }
+ Changed()
+}
+
+// A request to a group asks for the catalog's groups several times —
+// Resolve's Catalog, and Served for the members' images, the reply's
+// length and the window — and each built every group's entry again, from
+// every member of every group: the burst of CPU at once that a request to
+// a group cost with many providers (colmugx and Yayoi_no_yume on X). While
+// a request holds the catalog they are built once.
+func TestGroupEntriesBuiltOnceWhileHeld(t *testing.T) {
+ groupHeldFixture(t)
+ release := Hold()
+ defer release()
+ entries := providerEntries()
+ first := groupEntries(entries)
+ if len(first) < 2 {
+ t.Fatalf("the fixture has %d groups; the test needs the found ones too", len(first))
+ }
+ if again := groupEntries(entries); &again[0] != &first[0] {
+ t.Fatal("the groups' entries were built again under one Hold")
+ }
+ // what a caller is given is its own to change
+ c := Catalog()
+ c[0].Name = "changed"
+ if Catalog()[0].Name == "changed" || Served()[0].Name == "changed" {
+ t.Fatal("a change to Catalog's entries reached the next caller's")
+ }
+ // another catalog than the one held is built from, not answered for
+ other := append([]Entry(nil), entries[:len(entries)/2]...)
+ if got := groupEntries(other); len(got) > 0 && &got[0] == &first[0] {
+ t.Fatal("the groups of another catalog were the held one's")
+ }
+ // a write drops them
+ if err := SaveGroup(Group{ID: "g4", Name: "Renamed", Members: []string{"p0/m1", "p1/m1"}, Routing: Ordered}); err != nil {
+ t.Fatal(err)
+ }
+ found := false
+ for _, e := range Catalog() {
+ found = found || e.ID == GroupPrefix+"g4" && e.Name == "Renamed"
+ }
+ if !found {
+ t.Fatal("a group saved under the Hold kept its old entry")
+ }
+}
+
+// Each group's members (membersIn) asked whether the user disabled them,
+// and that parsed settings.json each time: a catalog of found groups
+// parsed it once a group, on each of the several looks a request makes.
+func TestGroupLooksParseNoSettingsPerGroupWhileHeld(t *testing.T) {
+ groupHeldFixture(t)
+ release := Hold()
+ defer release()
+ Served() // the first look builds what is held
+ const looks = 50
+ before := settings.Parses()
+ for range looks {
+ Served()
+ Groups()
+ if _, _, ok := FindGroup(GroupPrefix + "g4"); !ok {
+ t.Fatal("no group g4")
+ }
+ }
+ if n := settings.Parses() - before; n >= looks {
+ t.Fatalf("settings.json parsed %d times in %d looks at the groups under one Hold", n, looks)
+ }
+ // and a model disabled is still left out
+ for _, e := range Served() {
+ if e.ID == "p11/m9" {
+ t.Fatal("a disabled model was served")
+ }
+ }
+}
+
+// BenchmarkGroupEntries is the groups' entries of a catalog of 30
+// providers serving the same 60 models (60 groups magpie finds, of 30
+// members each), built afresh: what each request built several times.
+func BenchmarkGroupEntries(b *testing.B) {
+ isolate(b)
+ home := b.TempDir()
+ b.Setenv("HOME", home)
+ b.Setenv("USERPROFILE", home)
+ b.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config"))
+ b.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache"))
+ b.Setenv("MAGPIE_NO_ACCOUNTS", "1")
+ var models []string
+ for i := range 60 {
+ models = append(models, fmt.Sprintf("m%d", i))
+ }
+ for i := range 30 {
+ if err := Save(Provider{ID: fmt.Sprintf("p%d", i), Name: fmt.Sprintf("P%d", i), Key: "k", Models: models, Anthropic: "http://127.0.0.1:9"}); err != nil {
+ b.Fatal(err)
+ }
+ }
+ Changed()
+ entries := providerEntries()
+ b.ResetTimer()
+ for range b.N {
+ buildGroupEntries(entries)
+ }
+}
diff --git a/internal/provider/hold.go b/internal/provider/hold.go
index e5ec66347..48b2736f7 100644
--- a/internal/provider/hold.go
+++ b/internal/provider/hold.go
@@ -116,6 +116,50 @@ func heldPlugins() []plugin.Provider {
return heldOf("plugins", func() []plugin.Provider { return slices.Clip(plugin.Cached()) })
}
+// fromEntries is what was built from a catalog, and which catalog: its
+// first entry and its length.
+type fromEntries[T any] struct {
+ at *Entry
+ n int
+ v T
+}
+
+// heldFrom is heldOf for what is built from entries: held while a request
+// holds the catalog, and built again for another catalog than it was built
+// from (providerEntries built afresh after heldFor, or one a caller made).
+// A request asks for the groups' entries several times (Catalog, Served),
+// and each built them from every member of every group again (the CPU a
+// request to a group took at once, with many providers).
+func heldFrom[T any](key string, entries []Entry, build func([]Entry) T) T {
+ var at *Entry
+ if len(entries) > 0 {
+ at = &entries[0]
+ }
+ held.Lock()
+ if held.holds == 0 {
+ held.Unlock()
+ return build(entries)
+ }
+ if h, ok := held.built[key]; ok && time.Since(h.at) < heldFor {
+ if f := h.v.(fromEntries[T]); f.at == at && f.n == len(entries) {
+ held.Unlock()
+ return f.v
+ }
+ }
+ gen := held.gen
+ held.Unlock()
+ v := build(entries)
+ held.Lock()
+ if held.holds > 0 && held.gen == gen {
+ if held.built == nil {
+ held.built = map[string]heldValue{}
+ }
+ held.built[key] = heldValue{fromEntries[T]{at, len(entries), v}, time.Now()}
+ }
+ held.Unlock()
+ return v
+}
+
// heldSettings is the settings, read once while a request holds the
// catalog: a look at the agents asks them for every model of every agent.
// Only for reading — what it gives is shared by the request's look-ups.
diff --git a/internal/provider/models.go b/internal/provider/models.go
index 9d2060d3c..be33d6441 100644
--- a/internal/provider/models.go
+++ b/internal/provider/models.go
@@ -2282,7 +2282,7 @@ type Entry struct {
// provider not kept unlisted. A provider switched off has none in it.
func Catalog() []Entry {
entries := providerEntries()
- out := groupEntries(entries)
+ out := slices.Clone(groupEntries(entries)) // held: the caller's to change
kept := unlistedByGroups(entries)
for _, e := range entries {
if !e.Provider.Unlisted && !kept[e.ID] {
@@ -2321,7 +2321,7 @@ func unlistedByGroups(entries []Entry) map[string]bool {
// model a routing group can be made of, or a request can name.
func Served() []Entry {
entries := providerEntries()
- return append(groupEntries(entries), entries...)
+ return append(slices.Clone(groupEntries(entries)), entries...)
}
// Unlisted are the models Served has and Catalog doesn't: those of the
@@ -2522,8 +2522,10 @@ func Resolve(id string) (Provider, string, bool) {
}
func resolveIn(entries []Entry, id string) (Provider, string, bool) {
- for _, e := range entries {
- if e.ID == id {
+ // by index: an Entry, its Provider in it, is large, and a copy of each
+ // one looked past was most of a group's members resolved (membersIn)
+ for i := range entries {
+ if e := &entries[i]; e.ID == id {
return e.Provider, e.Model, true
}
}
@@ -2534,15 +2536,11 @@ func resolveIn(entries []Entry, id string) (Provider, string, bool) {
return *p, model, true
}
}
- var hits []Entry
- for _, e := range entries {
- if e.Model == id {
- hits = append(hits, e)
+ for i := range entries {
+ if e := &entries[i]; e.Model == id {
+ return e.Provider, e.Model, true // the first that serves it
}
}
- if len(hits) >= 1 {
- return hits[0].Provider, hits[0].Model, true
- }
// not exposed, but some provider lists it
var found []Provider
for _, p := range heldOf("listed", listedBy)[id] {