Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
provider: a request to a routing group builds the groups' entries once, not once per look at every member of every group (colmugx, Yayoi_no_yume on X)
yetonea595f1559c35
diff --git a/internal/gateway/group_overhead_bench_test.go b/internal/gateway/group_overhead_bench_test.gonew file mode 100644index 000000000..9540b1127--- /dev/null+++ b/internal/gateway/group_overhead_bench_test.go@@ -0,0 +1,196 @@+package gateway++import (+ "bytes"+ "fmt"+ "io"+ "io/fs"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "testing"+ "time"++ "github.com/yetone/magpie/internal/provider"+ "github.com/yetone/magpie/internal/settings"+)++// groupBenchUpstream answers an Anthropic request as it asked: streamed+// when its body says so. Refusing, it answers code with an overloaded+// error.+type groupBenchUpstream struct{ code int }++const groupBenchReply = `{"id":"msg_1","type":"message","role":"assistant","model":"m1","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":7,"output_tokens":1}}`++var groupBenchStream = sse(+ `event: message_start`+"\n"+`data: {"type":"message_start","message":{"id":"msg_1","model":"m1","usage":{"input_tokens":7}}}`,+ `event: content_block_start`+"\n"+`data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`,+ `event: content_block_delta`+"\n"+`data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ok"}}`,+ `event: content_block_stop`+"\n"+`data: {"type":"content_block_stop","index":0}`,+ `event: message_delta`+"\n"+`data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1}}`,+ `event: message_stop`+"\n"+`data: {"type":"message_stop"}`)++func (u groupBenchUpstream) ServeHTTP(w http.ResponseWriter, r *http.Request) {+ b, _ := io.ReadAll(r.Body)+ if u.code != 0 {+ w.Header().Set("Content-Type", "application/json")+ w.Header().Set("Retry-After", "3600")+ w.WriteHeader(u.code)+ io.WriteString(w, `{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}`)+ return+ }+ if bytes.Contains(b, []byte(`"stream":true`)) {+ w.Header().Set("Content-Type", "text/event-stream")+ io.WriteString(w, groupBenchStream)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ io.WriteString(w, groupBenchReply)+}++// groupBenchCatalog is a catalog of providers serving models each, the+// same model ids at each (so magpie finds a group of each model), and+// groups of one, four and eight members, one whose first member fails+// every time, and one whose first member is resting.+func groupBenchCatalog(b *testing.B, providers, models int) {+ b.Helper()+ b.Setenv("XDG_CONFIG_HOME", b.TempDir())+ b.Setenv("XDG_CACHE_HOME", b.TempDir())+ good := httptest.NewServer(groupBenchUpstream{})+ b.Cleanup(good.Close)+ failing := httptest.NewServer(groupBenchUpstream{code: 500})+ b.Cleanup(failing.Close)+ resting := httptest.NewServer(groupBenchUpstream{code: 429})+ b.Cleanup(resting.Close)+ var ids []string+ for i := range models {+ ids = append(ids, fmt.Sprintf("m%d", i))+ }+ for i := range providers {+ p := provider.Provider{ID: fmt.Sprintf("p%d", i), Name: fmt.Sprintf("P%d", i), Key: "k", Models: ids, Anthropic: good.URL}+ if err := provider.Save(p); err != nil {+ b.Fatal(err)+ }+ }+ for _, p := range []provider.Provider{+ {ID: "bad", Name: "Bad", Key: "k", Models: ids, Anthropic: failing.URL},+ {ID: "rest", Name: "Rest", Key: "k", Models: ids, Anthropic: resting.URL},+ } {+ if err := provider.Save(p); err != nil {+ b.Fatal(err)+ }+ }+ if err := settings.Save(settings.Settings{}); err != nil {+ b.Fatal(err)+ }+ for _, g := range []provider.Group{+ {ID: "g1", Name: "G1", Members: []string{"p0/m1"}, Routing: provider.Ordered},+ {ID: "g4", Name: "G4", Members: []string{"p0/m1", "p1/m1", "p2/m1", "p3/m1"}, Routing: provider.Ordered},+ {ID: "g8u", Name: "G8U", Members: []string{"p0/m1", "p1/m1", "p2/m1", "p3/m1", "p4/m1", "p5/m1", "p6/m1", "p7/m1"}, Routing: provider.LeastUsed},+ {ID: "gfail", Name: "GFail", Members: []string{"bad/m1", "p0/m1", "p1/m1"}, Routing: provider.Ordered},+ {ID: "grest", Name: "GRest", Members: []string{"rest/m1", "p0/m1", "p1/m1"}, Routing: provider.Ordered},+ } {+ if err := provider.SaveGroup(g); err != nil {+ b.Fatal(err)+ }+ }+ // as a user's files are: written a while ago, not just now (a file+ // written in the last seconds is read again each time)+ old := time.Now().Add(-time.Hour)+ filepath.WalkDir(os.Getenv("XDG_CONFIG_HOME"), func(path string, _ fs.DirEntry, _ error) error {+ return os.Chtimes(path, old, old)+ })+ provider.Changed()+}++// groupBenchCase is one benchmark's request.+type groupBenchCase struct {+ providers, models, words int+ stream bool+ model string+}++func (c groupBenchCase) name() string {+ return fmt.Sprintf("catalog=%dx%d/%dKB/stream=%v/%s", c.providers, c.models, c.words*4/1000, c.stream, c.model)+}++// groupBenchCases: a dozen providers of 20 models, with a ~200KB and a+// ~2MB conversation, and a large catalog (30 providers of 60 models, as+// a user of several relays has) with the ~200KB one.+func groupBenchCases() []groupBenchCase {+ var out []groupBenchCase+ for _, size := range []struct{ providers, models, words int }{{12, 20, 50_000}, {12, 20, 500_000}, {30, 60, 50_000}} {+ for _, stream := range []bool{false, true} {+ for _, model := range []string{"p0/m1", "group/g1", "group/g4", "group/g8u", "group/gfail", "group/grest"} {+ out = append(out, groupBenchCase{size.providers, size.models, size.words, stream, model})+ }+ }+ }+ return out+}++// groupBenchServer is the gateway with c's catalog, and a request of the+// body c sends that fails the benchmark unless it is answered.+func groupBenchServer(b *testing.B, c groupBenchCase) func() {+ noRests()+ b.Cleanup(noRests)+ groupBenchCatalog(b, c.providers, c.models)+ body := largeConversation(provider.Anthropic, c.model, c.words)+ if c.stream {+ body = bytes.Replace(body, []byte(`"stream":false`), []byte(`"stream":true`), 1)+ }+ b.SetBytes(int64(len(body)))+ h := New().Handler()+ do := func() {+ rec := httptest.NewRecorder()+ h.ServeHTTP(rec, httptest.NewRequest("POST", "/v1/messages", bytes.NewReader(body)))+ if rec.Code != 200 || !bytes.Contains(rec.Body.Bytes(), []byte("ok")) {+ b.Fatalf("%d %s", rec.Code, rec.Body.String())+ }+ }+ do() // the resting member rests+ return do+}++// BenchmarkGroupOverhead is a request to a provider's model and the same+// request to routing groups, streamed and not, with a long conversation:+// what magpie adds for a group is the difference. cpu-ms/op is the CPU+// the process spent, which a busy machine doesn't change as it does the+// wall time.+func BenchmarkGroupOverhead(b *testing.B) {+ for _, c := range groupBenchCases() {+ b.Run(c.name(), func(b *testing.B) {+ do := groupBenchServer(b, c)+ b.ResetTimer()+ cpu, _ := processCPU()+ for range b.N {+ do()+ }+ after, _ := processCPU()+ b.ReportMetric(float64(after-cpu)/float64(b.N)/1e6, "cpu-ms/op")+ })+ }+}++// BenchmarkGroupOverheadParallel is BenchmarkGroupOverhead's requests sent+// at once, as several agents send them.+func BenchmarkGroupOverheadParallel(b *testing.B) {+ for _, c := range groupBenchCases() {+ if c.stream || c.words > 50_000 {+ continue+ }+ b.Run(c.name(), func(b *testing.B) {+ do := groupBenchServer(b, c)+ b.ResetTimer()+ cpu, _ := processCPU()+ b.RunParallel(func(pb *testing.PB) {+ for pb.Next() {+ do()+ }+ })+ after, _ := processCPU()+ b.ReportMetric(float64(after-cpu)/float64(b.N)/1e6, "cpu-ms/op")+ })+ }+}diff --git a/internal/provider/account_test.go b/internal/provider/account_test.goindex 3230a4d2c..af677c910 100644--- a/internal/provider/account_test.go+++ b/internal/provider/account_test.go@@ -68,7 +68,7 @@ func signIn(t *testing.T) string {// isolate keeps tests off the machine's own Claude Code Keychain// logins and forgets anything a previous test cached.-func isolate(t *testing.T) {+func isolate(t testing.TB) {t.Helper()oldKeychain, oldBase, oldExe := claudeKeychain, claudeBase, claudeExecutableoldDevin := DevinExecutablediff --git a/internal/provider/disabled.go b/internal/provider/disabled.goindex b35585807..724d14f98 100644--- a/internal/provider/disabled.go+++ b/internal/provider/disabled.go@@ -34,6 +34,22 @@ var lastDisabled struct {// disabledList is the patterns in force, and unknown when settings.json// couldn't be read and none was read before.func disabledList() (list []string, unknown bool) {+ // read once while a request holds the catalog: each group's members+ // (membersIn) asked for it, and parsed settings.json each time+ r := heldOf("disabled", func() disabledRead {+ list, unknown := readDisabled()+ return disabledRead{slices.Clip(list), unknown}+ })+ return r.list, r.unknown+}++type disabledRead struct {+ list []string+ unknown bool+}++// readDisabled is disabledList read from settings.json now.+func readDisabled() (list []string, unknown bool) {s, err := settings.LoadChecked()lastDisabled.Lock()defer lastDisabled.Unlock()diff --git a/internal/provider/group.go b/internal/provider/group.goindex d641b5dd0..feba98c31 100644--- a/internal/provider/group.go+++ b/internal/provider/group.go@@ -868,10 +868,34 @@ func membersOff(entries []Entry, all []Group, g Group, off func(pid, model strin// every member fixed, the group offers the levels they are fixed at, so// that an agent still asks it to reason. A group that names its own// levels (Group.Levels) offers those, and so does a group in it for its-// models.+// models. Held while a request holds the catalog (heldFrom).func groupEntries(entries []Entry) []Entry {+ return heldFrom("group entries", entries, buildGroupEntries)+}++// entryOf is where in entries the last entry of provider pid's model is,+// -1 when none is.+type entryOf func(pid, model string) int++// entriesBy is entryOf for entries, looked up in a map built once rather+// than by a pass over every entry for each member of each group.+func entriesBy(entries []Entry) entryOf {+ at := make(map[[2]string]int, len(entries))+ for i := range entries {+ at[[2]string{entries[i].Provider.ID, entries[i].Model}] = i+ }+ return func(pid, model string) int {+ if i, ok := at[[2]string{pid, model}]; ok {+ return i+ }+ return -1+ }+}++func buildGroupEntries(entries []Entry) []Entry {var out []Entryall := groupsIn(entries)+ find := entriesBy(entries)for _, g := range all {if g.Hidden || g.Disabled {continue@@ -899,11 +923,10 @@ func groupEntries(entries []Entry) []Entry {images, thinks, ctx, output := false, false, 0, 0var imageInput *boolunknown, reasonless := false, false- for _, x := range entries {- if x.Provider.ID == m.Provider.ID && x.Model == m.Model {- efforts, images, thinks, ctx, output, imageInput = x.Efforts, x.Images, x.Reasoning, x.Context, x.Output, x.ImageInput- unknown, reasonless = levelsUnknown(x), x.Reasonless- }+ if j := find(m.Provider.ID, m.Model); j >= 0 {+ x := &entries[j]+ efforts, images, thinks, ctx, output, imageInput = x.Efforts, x.Images, x.Reasoning, x.Context, x.Output, x.ImageInput+ unknown, reasonless = levelsUnknown(*x), x.Reasonless}// it reasons when a member does: the gateway sends the// effort to the members that take it, and a member known not@@ -1011,8 +1034,13 @@ func groupEntries(entries []Entry) []Entry {ruledEntry(&e, g.Live(), ms, entries)if g.Context > 0 {e.Context, e.WindowSet = g.Context, true- } else if smallest, typed := smallestWindow(entries, ms); g.Context == ContextSmallest && smallest > 0 {- e.Context, e.WindowSet = smallest, typed+ } else if g.Context == ContextSmallest {+ // only for a group that follows its smallest member: run for+ // every group, it was another pass over the catalog for each+ // member of each (8f4d091c6)+ if smallest, typed := smallestWindowBy(entries, find, ms); smallest > 0 {+ e.Context, e.WindowSet = smallest, typed+ }}e.Family = g.Familyout = append(out, e)@@ -1029,12 +1057,14 @@ func groupEntries(entries []Entry) []Entry {// group set to 500K itself wasn't). A tie counts as typed: the window is// the user's either way.func smallestWindow(entries []Entry, ms []Member) (n int, typed bool) {+ return smallestWindowBy(entries, entriesBy(entries), ms)+}++func smallestWindowBy(entries []Entry, find entryOf, ms []Member) (n int, typed bool) {for _, m := range ms {ctx, set := 0, false- for _, x := range entries {- if x.Provider.ID == m.Provider.ID && x.Model == m.Model {- ctx, set = x.Context, x.WindowSet- }+ if j := find(m.Provider.ID, m.Model); j >= 0 {+ ctx, set = entries[j].Context, entries[j].WindowSet}switch {case ctx <= 0:diff --git a/internal/provider/group_held_test.go b/internal/provider/group_held_test.gonew file mode 100644index 000000000..819447c43--- /dev/null+++ b/internal/provider/group_held_test.go@@ -0,0 +1,136 @@+package provider++import (+ "fmt"+ "path/filepath"+ "testing"++ "github.com/yetone/magpie/internal/settings"+)++// groupHeldFixture is a dozen providers serving the same models, so+// magpie finds a group of each, and a group of the user's of four.+func groupHeldFixture(t *testing.T) {+ t.Helper()+ isolate(t)+ home := t.TempDir()+ t.Setenv("HOME", home)+ t.Setenv("USERPROFILE", home)+ t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config"))+ t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache"))+ t.Setenv("MAGPIE_NO_ACCOUNTS", "1")+ var models []string+ for i := range 10 {+ models = append(models, fmt.Sprintf("m%d", i))+ }+ for i := range 12 {+ if err := Save(Provider{ID: fmt.Sprintf("p%d", i), Name: fmt.Sprintf("P%d", i), Key: "k", Models: models, Anthropic: "http://127.0.0.1:9"}); err != nil {+ t.Fatal(err)+ }+ }+ if err := settings.Save(settings.Settings{DisabledModels: []string{"p11/m9"}}); err != nil {+ t.Fatal(err)+ }+ if err := SaveGroup(Group{ID: "g4", Name: "G4", Members: []string{"p0/m1", "p1/m1", "p2/m1", "p3/m1"}, Routing: Ordered}); err != nil {+ t.Fatal(err)+ }+ Changed()+}++// A request to a group asks for the catalog's groups several times —+// Resolve's Catalog, and Served for the members' images, the reply's+// length and the window — and each built every group's entry again, from+// every member of every group: the burst of CPU at once that a request to+// a group cost with many providers (colmugx and Yayoi_no_yume on X). While+// a request holds the catalog they are built once.+func TestGroupEntriesBuiltOnceWhileHeld(t *testing.T) {+ groupHeldFixture(t)+ release := Hold()+ defer release()+ entries := providerEntries()+ first := groupEntries(entries)+ if len(first) < 2 {+ t.Fatalf("the fixture has %d groups; the test needs the found ones too", len(first))+ }+ if again := groupEntries(entries); &again[0] != &first[0] {+ t.Fatal("the groups' entries were built again under one Hold")+ }+ // what a caller is given is its own to change+ c := Catalog()+ c[0].Name = "changed"+ if Catalog()[0].Name == "changed" || Served()[0].Name == "changed" {+ t.Fatal("a change to Catalog's entries reached the next caller's")+ }+ // another catalog than the one held is built from, not answered for+ other := append([]Entry(nil), entries[:len(entries)/2]...)+ if got := groupEntries(other); len(got) > 0 && &got[0] == &first[0] {+ t.Fatal("the groups of another catalog were the held one's")+ }+ // a write drops them+ if err := SaveGroup(Group{ID: "g4", Name: "Renamed", Members: []string{"p0/m1", "p1/m1"}, Routing: Ordered}); err != nil {+ t.Fatal(err)+ }+ found := false+ for _, e := range Catalog() {+ found = found || e.ID == GroupPrefix+"g4" && e.Name == "Renamed"+ }+ if !found {+ t.Fatal("a group saved under the Hold kept its old entry")+ }+}++// Each group's members (membersIn) asked whether the user disabled them,+// and that parsed settings.json each time: a catalog of found groups+// parsed it once a group, on each of the several looks a request makes.+func TestGroupLooksParseNoSettingsPerGroupWhileHeld(t *testing.T) {+ groupHeldFixture(t)+ release := Hold()+ defer release()+ Served() // the first look builds what is held+ const looks = 50+ before := settings.Parses()+ for range looks {+ Served()+ Groups()+ if _, _, ok := FindGroup(GroupPrefix + "g4"); !ok {+ t.Fatal("no group g4")+ }+ }+ if n := settings.Parses() - before; n >= looks {+ t.Fatalf("settings.json parsed %d times in %d looks at the groups under one Hold", n, looks)+ }+ // and a model disabled is still left out+ for _, e := range Served() {+ if e.ID == "p11/m9" {+ t.Fatal("a disabled model was served")+ }+ }+}++// BenchmarkGroupEntries is the groups' entries of a catalog of 30+// providers serving the same 60 models (60 groups magpie finds, of 30+// members each), built afresh: what each request built several times.+func BenchmarkGroupEntries(b *testing.B) {+ isolate(b)+ home := b.TempDir()+ b.Setenv("HOME", home)+ b.Setenv("USERPROFILE", home)+ b.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config"))+ b.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache"))+ b.Setenv("MAGPIE_NO_ACCOUNTS", "1")+ var models []string+ for i := range 60 {+ models = append(models, fmt.Sprintf("m%d", i))+ }+ for i := range 30 {+ if err := Save(Provider{ID: fmt.Sprintf("p%d", i), Name: fmt.Sprintf("P%d", i), Key: "k", Models: models, Anthropic: "http://127.0.0.1:9"}); err != nil {+ b.Fatal(err)+ }+ }+ Changed()+ entries := providerEntries()+ b.ResetTimer()+ for range b.N {+ buildGroupEntries(entries)+ }+}diff --git a/internal/provider/hold.go b/internal/provider/hold.goindex e5ec66347..48b2736f7 100644--- a/internal/provider/hold.go+++ b/internal/provider/hold.go@@ -116,6 +116,50 @@ func heldPlugins() []plugin.Provider {return heldOf("plugins", func() []plugin.Provider { return slices.Clip(plugin.Cached()) })}+// fromEntries is what was built from a catalog, and which catalog: its+// first entry and its length.+type fromEntries[T any] struct {+ at *Entry+ n int+ v T+}++// heldFrom is heldOf for what is built from entries: held while a request+// holds the catalog, and built again for another catalog than it was built+// from (providerEntries built afresh after heldFor, or one a caller made).+// A request asks for the groups' entries several times (Catalog, Served),+// and each built them from every member of every group again (the CPU a+// request to a group took at once, with many providers).+func heldFrom[T any](key string, entries []Entry, build func([]Entry) T) T {+ var at *Entry+ if len(entries) > 0 {+ at = &entries[0]+ }+ held.Lock()+ if held.holds == 0 {+ held.Unlock()+ return build(entries)+ }+ if h, ok := held.built[key]; ok && time.Since(h.at) < heldFor {+ if f := h.v.(fromEntries[T]); f.at == at && f.n == len(entries) {+ held.Unlock()+ return f.v+ }+ }+ gen := held.gen+ held.Unlock()+ v := build(entries)+ held.Lock()+ if held.holds > 0 && held.gen == gen {+ if held.built == nil {+ held.built = map[string]heldValue{}+ }+ held.built[key] = heldValue{fromEntries[T]{at, len(entries), v}, time.Now()}+ }+ held.Unlock()+ return v+}+// heldSettings is the settings, read once while a request holds the// catalog: a look at the agents asks them for every model of every agent.// Only for reading — what it gives is shared by the request's look-ups.diff --git a/internal/provider/models.go b/internal/provider/models.goindex 9d2060d3c..be33d6441 100644--- a/internal/provider/models.go+++ b/internal/provider/models.go@@ -2282,7 +2282,7 @@ type Entry struct {// provider not kept unlisted. A provider switched off has none in it.func Catalog() []Entry {entries := providerEntries()- out := groupEntries(entries)+ out := slices.Clone(groupEntries(entries)) // held: the caller's to changekept := unlistedByGroups(entries)for _, e := range entries {if !e.Provider.Unlisted && !kept[e.ID] {@@ -2321,7 +2321,7 @@ func unlistedByGroups(entries []Entry) map[string]bool {// model a routing group can be made of, or a request can name.func Served() []Entry {entries := providerEntries()- return append(groupEntries(entries), entries...)+ return append(slices.Clone(groupEntries(entries)), entries...)}// Unlisted are the models Served has and Catalog doesn't: those of the@@ -2522,8 +2522,10 @@ func Resolve(id string) (Provider, string, bool) {}func resolveIn(entries []Entry, id string) (Provider, string, bool) {- for _, e := range entries {- if e.ID == id {+ // by index: an Entry, its Provider in it, is large, and a copy of each+ // one looked past was most of a group's members resolved (membersIn)+ for i := range entries {+ if e := &entries[i]; e.ID == id {return e.Provider, e.Model, true}}@@ -2534,15 +2536,11 @@ func resolveIn(entries []Entry, id string) (Provider, string, bool) {return *p, model, true}}- var hits []Entry- for _, e := range entries {- if e.Model == id {- hits = append(hits, e)+ for i := range entries {+ if e := &entries[i]; e.Model == id {+ return e.Provider, e.Model, true // the first that serves it}}- if len(hits) >= 1 {- return hits[0].Provider, hits[0].Model, true- }// not exposed, but some provider lists itvar found []Providerfor _, p := range heldOf("listed", listedBy)[id] {