Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

provider: Served is shared while a request holds the catalog, not copied for each look (#3725)

yetone1360689ef2e1
diff --git a/internal/gateway/fallback.go b/internal/gateway/fallback.go
index 0a60f1345..b9996687b 100644
--- a/internal/gateway/fallback.go
+++ b/internal/gateway/fallback.go
@@ -1428,7 +1428,9 @@ func windowIn(served func() []provider.Entry, p provider.Provider, model string)
if n := p.Contexts[model]; n > 0 {
return n
}
- for _, e := range served() {
+ cat := served()
+ for i := range cat {
+ e := &cat[i]
if e.Group == "" && e.Provider.ID == p.ID && (e.Model == model || e.ID == model) && e.Context > 0 {
return e.Context
}
diff --git a/internal/gateway/maxoutput.go b/internal/gateway/maxoutput.go
index 3e8b783fd..146338d92 100644
--- a/internal/gateway/maxoutput.go
+++ b/internal/gateway/maxoutput.go
@@ -16,7 +16,9 @@ import (
// known: its catalog entry's (the user's setting, else the vendor's list
// or models.dev).
func outputLimit(p provider.Provider, model string) int {
- for _, e := range provider.Served() {
+ cat := provider.Served()
+ for i := range cat {
+ e := &cat[i] // an Entry is ~1.3KB, the catalog hundreds
if e.Group == "" && e.Provider.ID == p.ID && (e.Model == model || e.ID == model) && e.Output > 0 {
return e.Output
}
@@ -32,7 +34,9 @@ func outputLimit(p provider.Provider, model string) int {
// as max_tokens times the price turned every turn away, and the field the
// editor says caps max_tokens did nothing (YaCo on Discord).
func userOutput(p provider.Provider, model string) int {
- for _, e := range provider.Served() {
+ cat := provider.Served()
+ for i := range cat {
+ e := &cat[i] // an Entry is ~1.3KB, the catalog hundreds
if e.Group == "" && e.Provider.ID == p.ID && (e.Model == model || e.ID == model) && e.OutputSet && e.Output > 0 {
return e.Output
}
@@ -48,7 +52,9 @@ func userOutput(p provider.Provider, model string) int {
// turn ran out of, Codex showing "Incomplete response returned, reason:
// max_output_tokens" (#1792, tensornull).
func askedOutput(p provider.Provider, model string) int {
- for _, e := range provider.Served() {
+ cat := provider.Served()
+ for i := range cat {
+ e := &cat[i] // an Entry is ~1.3KB, the catalog hundreds
if e.Group == "" && e.Provider.ID == p.ID && (e.Model == model || e.ID == model) {
return e.PublishedOutput()
}
diff --git a/internal/provider/group_held_test.go b/internal/provider/group_held_test.go
index 88828cdf7..a3f62fd54 100644
--- a/internal/provider/group_held_test.go
+++ b/internal/provider/group_held_test.go
@@ -39,6 +39,28 @@ func groupHeldFixture(t *testing.T) {
Changed()
}
+// A request asks Served for each model it may go to (the reply's length
+// twice, the window), and each was a copy of the whole catalog: ~380KB a
+// request with 12 providers of 20 models (#3725). While a request holds
+// the catalog it is one, shared for reading.
+func TestServedSharedWhileHeld(t *testing.T) {
+ groupHeldFixture(t)
+ release := Hold()
+ defer release()
+ first := Served()
+ if len(first) == 0 {
+ t.Fatal("nothing served")
+ }
+ if again := Served(); &again[0] != &first[0] {
+ t.Fatal("Served was copied again under one Hold")
+ }
+ // clipped: a caller's append is its own
+ grown := append(first, Entry{ID: "added"})
+ if &grown[0] == &first[0] || len(Served()) != len(first) {
+ t.Fatal("an append to Served reached the next caller's")
+ }
+}
+
// A request to a group asks for the catalog's groups several times —
// Resolve's Catalog, and Served for the members' images, the reply's
// length and the window — and each built every group's entry again, from
diff --git a/internal/provider/models.go b/internal/provider/models.go
index 3d17fa5de..346fd47de 100644
--- a/internal/provider/models.go
+++ b/internal/provider/models.go
@@ -2467,10 +2467,14 @@ func unlistedByGroups(entries []Entry) map[string]bool {
}
// Served is the catalog with the unlisted providers' models as well: every
-// model a routing group can be made of, or a request can name.
+// model a routing group can be made of, or a request can name. Like
+// providerEntries it is shared, only for reading: a request asks it for
+// each model it may go to, and a copy each time was ~380KB with 12
+// providers of 20 models (#3725).
func Served() []Entry {
- entries := providerEntries()
- return append(slices.Clone(groupEntries(entries)), entries...)
+ return heldFrom("served", providerEntries(), func(entries []Entry) []Entry {
+ return slices.Clip(append(slices.Clone(groupEntries(entries)), entries...))
+ })
}
// Unlisted are the models Served has and Catalog doesn't: those of the