Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
package gateway
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
"github.com/yetone/magpie/internal/access"
"github.com/yetone/magpie/internal/provider"
)
// A gateway key held to some models (#882) sees only those in the model
// lists, is refused any other before a provider is asked — by a bare
// name, through a group with one it may not use or as a fallback — while
// another key and magpie's own calls for it go on.
func TestGatewayKeyModelsHoldAKey(t *testing.T) {
fresh(t)
plan := &fake{t: t, ctype: "application/json", reply: `{"id":"from-plan","choices":[]}`}
spare := &fake{t: t, ctype: "application/json", reply: `{"id":"from-spare","choices":[]}`}
twoProviders(t, plan, spare)
keys, secrets := newCaller(t, "Held", "Free")
if _, err := access.Update("models-key", access.Change{Key: keys[0].ID, Models: []string{" plan/* ", "PLAN/*"}}); err != nil {
t.Fatal(err)
}
if ks, _ := access.List(); !slices.Equal(ks[len(ks)-2].Models, []string{"plan/*"}) {
t.Fatalf("stored %+v", ks[len(ks)-2])
}
for _, g := range []provider.Group{
{Name: "Both", Members: []string{"plan/m1", "spare/m2"}, Routing: provider.Ordered},
{Name: "Solo", Members: []string{"plan/m1"}, Routing: provider.Ordered},
} {
if err := provider.SaveGroup(g); err != nil {
t.Fatal(err)
}
}
h := New().Handler()
do := func(secret, method, path, body string) *httptest.ResponseRecorder {
r := httptest.NewRequest(method, path, strings.NewReader(body))
r.Header.Set("Authorization", "Bearer "+secret)
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
return w
}
listed := func(secret string) []string {
var l struct{ Data []struct{ ID string } }
w := do(secret, "GET", "/v1/models", "")
if err := json.Unmarshal(w.Body.Bytes(), &l); err != nil {
t.Fatal(w.Body.String())
}
var ids []string
for _, m := range l.Data {
ids = append(ids, m.ID)
}
return ids
}
if ids := listed(secrets[0]); !slices.Contains(ids, "plan/m1") || !slices.Contains(ids, "group/solo") || slices.Contains(ids, "spare/m2") || slices.Contains(ids, "group/both") {
t.Fatalf("held key lists %v", ids)
}
if ids := listed(secrets[1]); !slices.Contains(ids, "spare/m2") || !slices.Contains(ids, "group/both") {
t.Fatalf("free key lists %v", ids)
}
if w := do(secrets[0], "GET", "/v1beta/models", ""); w.Code != 200 || !strings.Contains(w.Body.String(), "m1") || strings.Contains(w.Body.String(), "m2") {
t.Fatalf("gemini list %d %s", w.Code, w.Body.String())
}
if w := do(secrets[0], "GET", "/v1/models/spare/m2", ""); w.Code == 200 {
t.Fatal("held key found a model it may not use", w.Body.String())
}
chat := func(secret, model string) *httptest.ResponseRecorder {
return do(secret, "POST", "/v1/chat/completions", `{"model":"`+model+`","messages":[{"role":"user","content":"hi"}]}`)
}
for _, model := range []string{"spare/m2", "m2", "group/both"} {
w := chat(secrets[0], model)
var e struct {
Error struct{ Message, Type string }
}
json.Unmarshal(w.Body.Bytes(), &e)
if w.Code != 403 || e.Error.Type != "permission_error" || !strings.Contains(e.Error.Message, `"Held"`) || !strings.Contains(e.Error.Message, "plan/*") {
t.Fatalf("%s: %d %s", model, w.Code, w.Body.String())
}
if rec := lastUsage(t); !rec.Rejected || rec.Status != 403 || rec.CallerKeyID != keys[0].ID {
t.Fatalf("%s recorded as %+v", model, rec)
}
}
if spare.calls != 0 || plan.calls != 0 {
t.Fatal("a refused request reached a provider", plan.calls, spare.calls)
}
// said in Anthropic's shape on its API
w := do(secrets[0], "POST", "/v1/messages", `{"model":"spare/m2","max_tokens":10,"messages":[{"role":"user","content":"hi"}]}`)
var a struct {
Type string
Error struct{ Type string }
}
json.Unmarshal(w.Body.Bytes(), &a)
if w.Code != 403 || a.Type != "error" || a.Error.Type != "permission_error" {
t.Fatal("anthropic refusal", w.Code, w.Body.String())
}
for _, model := range []string{"plan/m1", "m1", "group/solo"} {
if w := chat(secrets[0], model); w.Code != 200 || !strings.Contains(w.Body.String(), "from-plan") {
t.Fatalf("%s: %d %s", model, w.Code, w.Body.String())
}
}
if w := chat(secrets[1], "spare/m2"); w.Code != 200 || !strings.Contains(w.Body.String(), "from-spare") {
t.Fatal("a free key was held", w.Code, w.Body.String())
}
// magpie's own call for the key (a search, a picture described) goes on
r := httptest.NewRequest("POST", "/v1/chat/completions", strings.NewReader(`{"model":"spare/m2","messages":[{"role":"user","content":"hi"}]}`))
r = r.WithContext(magpieChose(r.Context()))
r.Header.Set("Authorization", "Bearer "+secrets[0])
rec := httptest.NewRecorder()
h.ServeHTTP(rec, r)
if rec.Code != 200 {
t.Fatal("magpie's own call was held", rec.Code, rec.Body.String())
}
// plan out of quota: its fallback is a model the key may not use
plan.code, plan.reply = 429, `{"error":{"message":"slow down"}}`
spareCalls := spare.calls
if w := chat(secrets[0], "plan/m1"); w.Code == 200 || spare.calls != spareCalls {
t.Fatal("a held key fell back to a model it may not use", w.Code, w.Body.String())
}
if w := chat(secrets[1], "plan/m1"); w.Code != 200 || !strings.Contains(w.Body.String(), "from-spare") {
t.Fatal("a free key's fallback", w.Code, w.Body.String())
}
// "all" takes it off
if _, err := access.Update("models-key", access.Change{Key: keys[0].ID}); err != nil {
t.Fatal(err)
}
plan.code, plan.reply = 0, `{"id":"from-plan","choices":[]}`
if w := chat(secrets[0], "spare/m2"); w.Code != 200 {
t.Fatal("a key let off is still held", w.Code, w.Body.String())
}
}
// A provider's id inside the asked model can't pass for another
// provider's: OpenRouter's "anthropic/x" isn't Anthropic's.
func TestGatewayKeyModelsMatchTheServingProvider(t *testing.T) {
who := access.Identity{KeyName: "k", Models: []string{"anthropic/*"}}
if modelAllowed(who, provider.Provider{ID: "openrouter"}, "anthropic/claude") {
t.Fatal("a model id passed for its provider")
}
if !modelAllowed(who, provider.Provider{ID: "anthropic"}, "claude") {
t.Fatal("anthropic's own")
}
if !modelAllowed(access.Identity{}, provider.Provider{ID: "x"}, "y") {
t.Fatal("a free key")
}
if membersAllowed(who, nil) || !membersAllowed(access.Identity{}, nil) {
t.Fatal("an empty group")
}
}
// A key held to a routing group it names, "group/<id>" (Magic_zero on
// Discord), is shown the group and may use it, every member of it going
// through it — the next one too when the first fails — but not the
// members asked by name, a group it doesn't name, or a fallback outside
// the group; "group/*" names every group, and a group named inside one
// it doesn't name counts for its own members.
func TestGatewayKeyModelsNameAGroup(t *testing.T) {
fresh(t)
plan := &fake{t: t, ctype: "application/json", reply: `{"id":"from-plan","choices":[]}`}
spare := &fake{t: t, ctype: "application/json", reply: `{"id":"from-spare","choices":[]}`}
twoProviders(t, plan, spare)
keys, secrets := newCaller(t, "Group", "Every group", "Inner")
for i, ms := range [][]string{{"group/both"}, {"group/*"}, {"group/solo", "spare/m2"}} {
if _, err := access.Update("models-key", access.Change{Key: keys[i].ID, Models: ms}); err != nil {
t.Fatal(err)
}
}
for _, g := range []provider.Group{
{Name: "Both", Members: []string{"plan/m1", "spare/m2"}, Routing: provider.Ordered},
{Name: "Solo", Members: []string{"plan/m1"}, Routing: provider.Ordered},
{Name: "Outer", Members: []string{"group/solo", "spare/m2"}, Routing: provider.Ordered},
} {
if err := provider.SaveGroup(g); err != nil {
t.Fatal(err)
}
}
h := New().Handler()
do := func(secret, method, path, body string) *httptest.ResponseRecorder {
r := httptest.NewRequest(method, path, strings.NewReader(body))
r.Header.Set("Authorization", "Bearer "+secret)
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
return w
}
listed := func(secret string) []string {
var l struct{ Data []struct{ ID string } }
w := do(secret, "GET", "/v1/models", "")
if err := json.Unmarshal(w.Body.Bytes(), &l); err != nil {
t.Fatal(w.Body.String())
}
var ids []string
for _, m := range l.Data {
ids = append(ids, m.ID)
}
slices.Sort(ids)
return ids
}
if ids := listed(secrets[0]); !slices.Equal(ids, []string{"group/both"}) {
t.Fatalf("a key naming group/both lists %v", ids)
}
if ids := listed(secrets[1]); !slices.Equal(ids, []string{"group/both", "group/outer", "group/solo"}) {
t.Fatalf("a key naming group/* lists %v", ids)
}
if ids := listed(secrets[2]); !slices.Contains(ids, "group/outer") || !slices.Contains(ids, "group/solo") || slices.Contains(ids, "plan/m1") || slices.Contains(ids, "group/both") {
t.Fatalf("a key naming group/solo and spare/m2 lists %v", ids)
}
chat := func(secret, model string) *httptest.ResponseRecorder {
return do(secret, "POST", "/v1/chat/completions", `{"model":"`+model+`","messages":[{"role":"user","content":"hi"}]}`)
}
for _, c := range []struct {
key int
model string
code int
from string
}{
{0, "group/both", 200, "from-plan"},
{0, "both", 200, "from-plan"},
{0, "plan/m1", 403, ""},
{0, "spare/m2", 403, ""},
{0, "group/solo", 403, ""},
{1, "group/solo", 200, "from-plan"},
{1, "plan/m1", 403, ""},
{2, "group/outer", 200, "from-plan"},
{2, "group/both", 403, ""},
} {
w := chat(secrets[c.key], c.model)
if w.Code != c.code || c.from != "" && !strings.Contains(w.Body.String(), c.from) {
t.Fatalf("key %d asking %s: %d %s", c.key, c.model, w.Code, w.Body.String())
}
}
// the group's first member out of quota: the next member, through it
plan.code, plan.reply = 429, `{"error":{"message":"slow down"}}`
if w := chat(secrets[0], "group/both"); w.Code != 200 || !strings.Contains(w.Body.String(), "from-spare") {
t.Fatal("a named group's next member", w.Code, w.Body.String())
}
// but plan's own fallback, spare/m2, isn't Solo's
spareCalls := spare.calls
if w := chat(secrets[1], "group/solo"); w.Code == 200 || spare.calls != spareCalls {
t.Fatal("a named group fell back outside itself", w.Code, w.Body.String())
}
}
// A gateway key held to some models (#882) counts no tokens on one it may
// not use — Anthropic's count_tokens and Gemini's :countTokens, a group
// judged by its members — and System One asks its decision model only of
// a key that may use it; a free key goes on as before, and no provider is
// asked for any of it.
func TestGatewayKeyModelsHoldCountTokensAndSystemOne(t *testing.T) {
fresh(t)
plan := &fake{t: t, ctype: "application/json", reply: `{"id":"from-plan","choices":[]}`}
spare := &fake{t: t, ctype: "application/json", reply: `{"id":"from-spare","choices":[]}`}
twoProviders(t, plan, spare)
var decided int
jev := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
decided++
w.Header().Set("Content-Type", "application/json")
io.WriteString(w, `{"answers":{"intent":{"choice":"bug"}},"usage":{"input_tokens":10,"output_tokens":2}}`)
}))
t.Cleanup(jev.Close)
if err := provider.Save(provider.Provider{ID: "jev", Name: "Jev", Key: "k", Decide: jev.URL + "/v1"}); err != nil {
t.Fatal(err)
}
if err := provider.SaveGroup(provider.Group{Name: "Both", Members: []string{"plan/m1", "spare/m2"}, Routing: provider.Ordered}); err != nil {
t.Fatal(err)
}
keys, secrets := newCaller(t, "Held", "Free")
if _, err := access.Update("models-key", access.Change{Key: keys[0].ID, Models: []string{"plan/*"}}); err != nil {
t.Fatal(err)
}
h := New().Handler()
do := func(secret, method, path, body string) *httptest.ResponseRecorder {
r := httptest.NewRequest(method, path, strings.NewReader(body))
r.Header.Set("Authorization", "Bearer "+secret)
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
return w
}
// counting tokens on a model it may not use, by id, bare and through a
// group with one in: said in Anthropic's own error shape
for _, model := range []string{"spare/m2", "m2", "group/both"} {
w := do(secrets[0], "POST", "/v1/messages/count_tokens", `{"model":"`+model+`","messages":[{"role":"user","content":"hi"}]}`)
var e struct {
Error struct{ Message, Type string }
}
json.Unmarshal(w.Body.Bytes(), &e)
if w.Code != 403 || e.Error.Type != "permission_error" || !strings.Contains(e.Error.Message, `"Held"`) || !strings.Contains(e.Error.Message, "plan/*") {
t.Fatalf("counting %s: %d %s", model, w.Code, w.Body.String())
}
}
// one it may use is counted, and a free key counts anything
for _, c := range []struct{ secret, model string }{
{secrets[0], "plan/m1"}, {secrets[1], "spare/m2"},
} {
if w := do(c.secret, "POST", "/v1/messages/count_tokens", `{"model":"`+c.model+`","messages":[{"role":"user","content":"hi"}]}`); w.Code != 200 || !strings.Contains(w.Body.String(), "input_tokens") {
t.Fatalf("counting %s: %d %s", c.model, w.Code, w.Body.String())
}
}
// Gemini's counting, the model in its URL, is held the same way
w := do(secrets[0], "POST", "/v1beta/models/spare/m2:countTokens", `{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`)
var g struct {
Error struct {
Code int
Status string
}
}
if w.Code != 403 || json.Unmarshal(w.Body.Bytes(), &g) != nil || g.Error.Code != 403 || g.Error.Status != "PERMISSION_DENIED" {
t.Fatalf("gemini counting: %d %s", w.Code, w.Body.String())
}
if w = do(secrets[0], "POST", "/v1beta/models/plan/m1:countTokens", `{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`); w.Code != 200 || !strings.Contains(w.Body.String(), "totalTokens") {
t.Fatalf("gemini counting allowed: %d %s", w.Code, w.Body.String())
}
if plan.calls != 0 || spare.calls != 0 || decided != 0 {
t.Fatal("a held count or decision reached a provider", plan.calls, spare.calls, decided)
}
// System One's decision model is a model: the held key is refused it,
// a free key asks, and one given the decider asks too
const ask = `{"model":"jev/jev-latest","state":{"message":"hi"},"questions":{"intent":{"type":"choice"}}}`
w = do(secrets[0], "POST", "/v1/systemone", ask)
var e struct {
Error struct{ Message, Type string }
}
json.Unmarshal(w.Body.Bytes(), &e)
if w.Code != 403 || e.Error.Type != "permission_error" || !strings.Contains(e.Error.Message, `"Held"`) {
t.Fatalf("system one held: %d %s", w.Code, w.Body.String())
}
if decided != 0 {
t.Fatal("a held decision reached its provider", decided)
}
if w = do(secrets[1], "POST", "/v1/systemone", ask); w.Code != 200 || decided != 1 {
t.Fatalf("a free key's decision: %d %s, decided %d", w.Code, w.Body.String(), decided)
}
if _, err := access.Update("models-key", access.Change{Key: keys[0].ID, Models: []string{"jev/*"}}); err != nil {
t.Fatal(err)
}
if w = do(secrets[0], "POST", "/v1/systemone", ask); w.Code != 200 || decided != 2 {
t.Fatalf("a key given the decider: %d %s, decided %d", w.Code, w.Body.String(), decided)
}
}
// Another magpie added as a provider with a gateway key held to some
// models asks this one for its image and video models too
// (provider.DrawersHeader, VideomakersHeader). It is told of those the
// key may use alone, as it is of chat models; a key free of models, and
// this computer, are told of every one.
func TestGatewayKeyModelsHoldTheImageAndVideoLists(t *testing.T) {
grokSignedIn(t)
newGrokMedia(t)
keys, secrets := newCaller(t, "Elsewhere", "Grok", "Free")
if _, err := access.Update("models-key", access.Change{Key: keys[0].ID, Models: []string{"elsewhere/*"}}); err != nil {
t.Fatal(err)
}
if _, err := access.Update("models-key", access.Change{Key: keys[1].ID, Models: []string{"grok/grok-imagine-video"}}); err != nil {
t.Fatal(err)
}
h := New().Handler()
listed := func(secret string) (image, video []string) {
r := httptest.NewRequest("GET", "/v1/models", nil)
if secret != "" {
r.Header.Set("Authorization", "Bearer "+secret)
}
r.Header.Set(provider.DrawersHeader, "1")
r.Header.Set(provider.VideomakersHeader, "1")
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
var l struct{ Data []struct{ ID, Kind string } }
if err := json.Unmarshal(w.Body.Bytes(), &l); err != nil {
t.Fatal(w.Body.String())
}
for _, m := range l.Data {
switch m.Kind {
case "image":
image = append(image, m.ID)
case "video":
video = append(video, m.ID)
}
}
return image, video
}
if image, video := listed(""); !slices.Contains(image, "grok/grok-imagine-image") || !slices.Contains(video, "grok/grok-imagine-video") {
t.Fatalf("this computer is told of image %v, video %v", image, video)
}
if image, video := listed(secrets[2]); !slices.Contains(image, "grok/grok-imagine-image") || !slices.Contains(video, "grok/grok-imagine-video") {
t.Fatalf("a free key is told of image %v, video %v", image, video)
}
if image, video := listed(secrets[0]); len(image) != 0 || len(video) != 0 {
t.Fatalf("a key held elsewhere is told of image %v, video %v", image, video)
}
if image, video := listed(secrets[1]); len(image) != 0 || !slices.Equal(video, []string{"grok/grok-imagine-video"}) {
t.Fatalf("a key held to one video model is told of image %v, video %v", image, video)
}
}
// A gateway key held to an account that serves only a chat model must not
// be told of that provider's image/video models in /v1/models (#2579): the
// chat catalog already applies accountServes; drawerObjects and
// videomakerObjects used to list them from modelAllowed alone.
func TestGatewayKeyAccountsHoldTheImageAndVideoLists(t *testing.T) {
grokSignedIn(t)
newGrokMedia(t)
// the plugin's one account, which grokSignedIn signs in, held to a
// chat model of its own
p, err := provider.Find("grok")
if err != nil || len(p.AccountRefs()) != 1 {
t.Fatalf("Grok's accounts: %v, %v", p.AccountRefs(), err)
}
ref := p.AccountRefs()[0]
keys, secrets := newCaller(t, "Held", "Free")
if _, err := access.Update("accounts-key", access.Change{Key: keys[0].ID, Accounts: []string{"grok/" + ref}}); err != nil {
t.Fatal(err)
}
if err := provider.SetAccountModels("grok", ref, []string{"fake-1"}); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = provider.SetAccountModels("grok", ref, nil)
})
h := New().Handler()
listed := func(secret string) (image, video []string) {
r := httptest.NewRequest("GET", "/v1/models", nil)
if secret != "" {
r.Header.Set("Authorization", "Bearer "+secret)
}
r.Header.Set(provider.DrawersHeader, "1")
r.Header.Set(provider.VideomakersHeader, "1")
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
var l struct{ Data []struct{ ID, Kind string } }
if err := json.Unmarshal(w.Body.Bytes(), &l); err != nil {
t.Fatal(w.Body.String())
}
for _, m := range l.Data {
switch m.Kind {
case "image":
image = append(image, m.ID)
case "video":
video = append(video, m.ID)
}
}
return image, video
}
if image, video := listed(""); !slices.Contains(image, "grok/grok-imagine-image") || !slices.Contains(video, "grok/grok-imagine-video") {
t.Fatalf("this computer is told of image %v, video %v", image, video)
}
if image, video := listed(secrets[1]); !slices.Contains(image, "grok/grok-imagine-image") || !slices.Contains(video, "grok/grok-imagine-video") {
t.Fatalf("a free key is told of image %v, video %v", image, video)
}
if image, video := listed(secrets[0]); len(image) != 0 || len(video) != 0 {
t.Fatalf("a key whose account serves no image or video model is told of image %v, video %v", image, video)
}
}